Merge bitcoindevkit/bdk#1282: Bump version to 1.0.0-alpha.4

d33acc1466 Bump version to 1.0.0-alpha.4 (Steve Myers)

Pull request description:

  ### Description

  Bump version to 1.0.0-alpha.4, also:

  bdk_chain to 0.8.0
  bdk_bitcoind_rpc to 0.3.0
  bdk_electrum to 0.6.0
  bdk_esplora to 0.6.0
  bdk_file_store to 0.4.0

  Fixes #1281.

  ### Notes to the reviewers

  Since the version of bdk_chain changed all crates that depend on it also need to have their versions bumped.

  ### Checklists

  #### All Submissions:

  * [x] I've signed all my commits
  * [x] I followed the [contribution guidelines](https://github.com/bitcoindevkit/bdk/blob/master/CONTRIBUTING.md)
  * [x] I ran `cargo fmt` and `cargo clippy` before committing

ACKs for top commit:
  evanlinjin:
    ACK d33acc1466

Tree-SHA512: ffb5c05b45e9d646ef796bd894353e4b58c3aae3dcf0921dfb9fcae45020ed4fc82274a7497494d6259bfffff9c2f97c530d2cbe6ace578babd04852af461f3a

.github/workflows/cont_integration.yml

@@ -33,7 +33,7 @@ jobs:
           cargo update -p zip --precise "0.6.2"
           cargo update -p time --precise "0.3.20"
           cargo update -p jobserver --precise "0.1.26"
-          cargo update -p reqwest --precise "0.11.19"
+          cargo update -p home --precise "0.5.5"
       - name: Build
         run: cargo build ${{ matrix.features }}
       - name: Test
@@ -118,9 +118,7 @@ jobs:
       - uses: actions/checkout@v1
       - uses: actions-rs/toolchain@v1
         with:
-            # we pin clippy instead of using "stable" so that our CI doesn't break
-            # at each new cargo release
-            toolchain: "1.67.0"
+            toolchain: stable
             components: clippy
             override: true
       - name: Rust Cache

Cargo.toml

@@ -7,6 +7,7 @@ members = [
     "crates/electrum",
     "crates/esplora",
     "crates/bitcoind_rpc",
+    "crates/hwi",
     "example-crates/example_cli",
     "example-crates/example_electrum",
     "example-crates/example_esplora",

README.md

@@ -65,14 +65,14 @@ This library should compile with any combination of features with Rust 1.63.0.
 To build with the MSRV you will need to pin dependencies as follows:
 
 ```shell
-# zip 0.6.3 has MSRV 1.64.0+
+# zip 0.6.3 has MSRV 1.64.0
 cargo update -p zip --precise "0.6.2"
-# time 0.3.21 has MSRV 1.65.0+
+# time 0.3.21 has MSRV 1.65.0
 cargo update -p time --precise "0.3.20"
 # jobserver 0.1.27 has MSRV 1.66.0
 cargo update -p jobserver --precise "0.1.26"
-# reqwest 0.11.20 has MSRV > 1.63.0+
-cargo update -p reqwest --precise "0.11.19"
+# home 0.5.9 has MSRV 1.70.0
+cargo update -p home --precise "0.5.5"
 ```
 
 ## License

clippy.toml

@@ -1 +1 @@
-msrv="1.57.0"
+msrv="1.63.0"

crates/bdk/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "bdk"
 homepage = "https://bitcoindevkit.org"
-version = "1.0.0-alpha.2"
+version = "1.0.0-alpha.4"
 repository = "https://github.com/bitcoindevkit/bdk"
 documentation = "https://docs.rs/bdk"
 description = "A modern, lightweight, descriptor-based wallet library"
@@ -18,11 +18,10 @@ miniscript = { version = "10.0.0", features = ["serde"], default-features = fals
 bitcoin = { version = "0.30.0", features = ["serde", "base64", "rand-std"], default-features = false }
 serde = { version = "^1.0", features = ["derive"] }
 serde_json = { version = "^1.0" }
-bdk_chain = { path = "../chain", version = "0.6.0", features = ["miniscript", "serde"], default-features = false }
+bdk_chain = { path = "../chain", version = "0.8.0", features = ["miniscript", "serde"], default-features = false }
 
 # Optional dependencies
-hwi = { version = "0.7.0", optional = true, features = [ "miniscript"] }
-bip39 = { version = "1.0.1", optional = true }
+bip39 = { version = "2.0", optional = true }
 
 [target.'cfg(target_arch = "wasm32")'.dependencies]
 getrandom = "0.2"
@@ -34,8 +33,6 @@ std = ["bitcoin/std", "miniscript/std", "bdk_chain/std"]
 compiler = ["miniscript/compiler"]
 all-keys = ["keys-bip39"]
 keys-bip39 = ["bip39"]
-hardware-signer = ["hwi"]
-test-hardware-signer = ["hardware-signer"]
 
 # This feature is used to run `cargo check` in our CI targeting wasm. It's not recommended
 # for libraries to explicitly include the "getrandom/js" feature, so we only do it when

crates/bdk/examples/mnemonic_to_descriptors.rs

@@ -33,8 +33,8 @@ fn main() -> Result<(), anyhow::Error> {
     let mnemonic_with_passphrase = (mnemonic, None);
 
     // define external and internal derivation key path
-    let external_path = DerivationPath::from_str("m/86h/0h/0h/0").unwrap();
-    let internal_path = DerivationPath::from_str("m/86h/0h/0h/1").unwrap();
+    let external_path = DerivationPath::from_str("m/86h/1h/0h/0").unwrap();
+    let internal_path = DerivationPath::from_str("m/86h/1h/0h/1").unwrap();
 
     // generate external and internal descriptor from mnemonic
     let (external_descriptor, ext_keymap) =

crates/bdk/src/descriptor/template.rs

@@ -575,7 +575,7 @@ mod test {
 
         if let ExtendedDescriptor::Pkh(pkh) = xdesc.0 {
             let path: Vec<ChildNumber> = pkh.into_inner().full_derivation_path().unwrap().into();
-            let purpose = path.get(0).unwrap();
+            let purpose = path.first().unwrap();
             assert_matches!(purpose, Hardened { index: 44 });
             let coin_type = path.get(1).unwrap();
             assert_matches!(coin_type, Hardened { index: 0 });
@@ -589,7 +589,7 @@ mod test {
 
         if let ExtendedDescriptor::Pkh(pkh) = tdesc.0 {
             let path: Vec<ChildNumber> = pkh.into_inner().full_derivation_path().unwrap().into();
-            let purpose = path.get(0).unwrap();
+            let purpose = path.first().unwrap();
             assert_matches!(purpose, Hardened { index: 44 });
             let coin_type = path.get(1).unwrap();
             assert_matches!(coin_type, Hardened { index: 1 });

crates/bdk/src/lib.rs

@@ -17,8 +17,6 @@ extern crate std;
 pub extern crate alloc;
 
 pub extern crate bitcoin;
-#[cfg(feature = "hardware-signer")]
-pub extern crate hwi;
 pub extern crate miniscript;
 extern crate serde;
 extern crate serde_json;

crates/bdk/src/wallet/mod.rs

@@ -50,10 +50,6 @@ pub mod tx_builder;
 pub(crate) mod utils;
 
 pub mod error;
-#[cfg(feature = "hardware-signer")]
-#[cfg_attr(docsrs, doc(cfg(feature = "hardware-signer")))]
-pub mod hardwaresigner;
-
 pub use utils::IsDust;
 
 #[allow(deprecated)]
@@ -237,6 +233,7 @@ impl Wallet {
         network: Network,
     ) -> Result<Self, DescriptorError> {
         Self::new(descriptor, change_descriptor, (), network).map_err(|e| match e {
+            NewError::NonEmptyDatabase => unreachable!("mock-database cannot have data"),
             NewError::Descriptor(e) => e,
             NewError::Write(_) => unreachable!("mock-write must always succeed"),
         })
@@ -251,6 +248,7 @@ impl Wallet {
     ) -> Result<Self, crate::descriptor::DescriptorError> {
         Self::new_with_genesis_hash(descriptor, change_descriptor, (), network, genesis_hash)
             .map_err(|e| match e {
+                NewError::NonEmptyDatabase => unreachable!("mock-database cannot have data"),
                 NewError::Descriptor(e) => e,
                 NewError::Write(_) => unreachable!("mock-write must always succeed"),
             })
@@ -264,6 +262,11 @@ where
     /// Infallibly return a derived address using the external descriptor, see [`AddressIndex`] for
     /// available address index selection strategies. If none of the keys in the descriptor are derivable
     /// (i.e. does not end with /*) then the same address will always be returned for any [`AddressIndex`].
+    ///
+    /// # Panics
+    ///
+    /// This panics when the caller requests for an address of derivation index greater than the
+    /// BIP32 max index.
     pub fn get_address(&mut self, address_index: AddressIndex) -> AddressInfo {
         self.try_get_address(address_index).unwrap()
     }
@@ -275,6 +278,11 @@ where
     /// see [`AddressIndex`] for available address index selection strategies. If none of the keys
     /// in the descriptor are derivable (i.e. does not end with /*) then the same address will always
     /// be returned for any [`AddressIndex`].
+    ///
+    /// # Panics
+    ///
+    /// This panics when the caller requests for an address of derivation index greater than the
+    /// BIP32 max index.
     pub fn get_internal_address(&mut self, address_index: AddressIndex) -> AddressInfo {
         self.try_get_internal_address(address_index).unwrap()
     }
@@ -288,6 +296,8 @@ where
 /// [`new_with_genesis_hash`]: Wallet::new_with_genesis_hash
 #[derive(Debug)]
 pub enum NewError<W> {
+    /// Database already has data.
+    NonEmptyDatabase,
     /// There was problem with the passed-in descriptor(s).
     Descriptor(crate::descriptor::DescriptorError),
     /// We were unable to write the wallet's data to the persistence backend.
@@ -300,6 +310,10 @@ where
 {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         match self {
+            NewError::NonEmptyDatabase => write!(
+                f,
+                "database already has data - use `load` or `new_or_load` methods instead"
+            ),
             NewError::Descriptor(e) => e.fmt(f),
             NewError::Write(e) => e.fmt(f),
         }
@@ -348,7 +362,7 @@ where
 #[cfg(feature = "std")]
 impl<L> std::error::Error for LoadError<L> where L: core::fmt::Display + core::fmt::Debug {}
 
-/// Error type for when we try load a [`Wallet`] from persistence and creating it if non-existant.
+/// Error type for when we try load a [`Wallet`] from persistence and creating it if non-existent.
 ///
 /// Methods [`new_or_load`] and [`new_or_load_with_genesis_hash`] may return this error.
 ///
@@ -446,13 +460,18 @@ impl<D> Wallet<D> {
     pub fn new_with_genesis_hash<E: IntoWalletDescriptor>(
         descriptor: E,
         change_descriptor: Option<E>,
-        db: D,
+        mut db: D,
         network: Network,
         genesis_hash: BlockHash,
     ) -> Result<Self, NewError<D::WriteError>>
     where
         D: PersistBackend<ChangeSet>,
     {
+        if let Ok(changeset) = db.load_from_persistence() {
+            if changeset.is_some() {
+                return Err(NewError::NonEmptyDatabase);
+            }
+        }
         let secp = Secp256k1::new();
         let (chain, chain_changeset) = LocalChain::from_genesis_hash(genesis_hash);
         let mut index = KeychainTxOutIndex::<KeychainKind>::default();
@@ -517,7 +536,9 @@ impl<D> Wallet<D> {
             create_signers(&mut index, &secp, descriptor, change_descriptor, network)
                 .map_err(LoadError::Descriptor)?;
 
-        let indexed_graph = IndexedTxGraph::new(index);
+        let mut indexed_graph = IndexedTxGraph::new(index);
+        indexed_graph.apply_changeset(changeset.indexed_tx_graph);
+
         let persist = Persist::new(db);
 
         Ok(Wallet {
@@ -613,6 +634,9 @@ impl<D> Wallet<D> {
                 genesis_hash,
             )
             .map_err(|e| match e {
+                NewError::NonEmptyDatabase => {
+                    unreachable!("database is already checked to have no data")
+                }
                 NewError::Descriptor(e) => NewOrLoadError::Descriptor(e),
                 NewError::Write(e) => NewOrLoadError::Write(e),
             }),
@@ -635,6 +659,11 @@ impl<D> Wallet<D> {
     ///
     /// A `PersistBackend<ChangeSet>::WriteError` will result if unable to persist the new address
     /// to the `PersistBackend`.
+    ///
+    /// # Panics
+    ///
+    /// This panics when the caller requests for an address of derivation index greater than the
+    /// BIP32 max index.
     pub fn try_get_address(
         &mut self,
         address_index: AddressIndex,
@@ -655,6 +684,11 @@ impl<D> Wallet<D> {
     /// see [`AddressIndex`] for available address index selection strategies. If none of the keys
     /// in the descriptor are derivable (i.e. does not end with /*) then the same address will always
     /// be returned for any [`AddressIndex`].
+    ///
+    /// # Panics
+    ///
+    /// This panics when the caller requests for an address of derivation index greater than the
+    /// BIP32 max index.
     pub fn try_get_internal_address(
         &mut self,
         address_index: AddressIndex,
@@ -677,6 +711,11 @@ impl<D> Wallet<D> {
     /// See [`AddressIndex`] for available address index selection strategies. If none of the keys
     /// in the descriptor are derivable (i.e. does not end with /*) then the same address will
     /// always be returned for any [`AddressIndex`].
+    ///
+    /// # Panics
+    ///
+    /// This panics when the caller requests for an address of derivation index greater than the
+    /// BIP32 max index.
     fn _get_address(
         &mut self,
         keychain: KeychainKind,
@@ -696,12 +735,14 @@ impl<D> Wallet<D> {
                 let ((index, spk), index_changeset) = txout_index.next_unused_spk(&keychain);
                 (index, spk.into(), Some(index_changeset))
             }
-            AddressIndex::Peek(index) => {
-                let (index, spk) = txout_index
-                    .spks_of_keychain(&keychain)
-                    .take(index as usize + 1)
-                    .last()
-                    .unwrap();
+            AddressIndex::Peek(mut peek_index) => {
+                let mut spk_iter = txout_index.unbounded_spk_iter(&keychain);
+                if !spk_iter.descriptor().has_wildcard() {
+                    peek_index = 0;
+                }
+                let (index, spk) = spk_iter
+                    .nth(peek_index as usize)
+                    .expect("derivation index is out of bounds");
                 (index, spk, None)
             }
         };
@@ -731,7 +772,7 @@ impl<D> Wallet<D> {
     ///
     /// Will only return `Some(_)` if the wallet has given out the spk.
     pub fn derivation_of_spk(&self, spk: &Script) -> Option<(KeychainKind, u32)> {
-        self.indexed_graph.index.index_of_spk(spk).copied()
+        self.indexed_graph.index.index_of_spk(spk)
     }
 
     /// Return the list of unspent outputs of this wallet
@@ -770,7 +811,7 @@ impl<D> Wallet<D> {
         self.chain.tip()
     }
 
-    /// Returns a iterators of all the script pubkeys for the `Internal` and External` variants in `KeychainKind`.
+    /// Get unbounded script pubkey iterators for both `Internal` and `External` keychains.
     ///
     /// This is intended to be used when doing a full scan of your addresses (e.g. after restoring
     /// from seed words). You pass the `BTreeMap` of iterators to a blockchain data source (e.g.
@@ -778,36 +819,36 @@ impl<D> Wallet<D> {
     ///
     /// Note carefully that iterators go over **all** script pubkeys on the keychains (not what
     /// script pubkeys the wallet is storing internally).
-    pub fn spks_of_all_keychains(
+    pub fn all_unbounded_spk_iters(
         &self,
     ) -> BTreeMap<KeychainKind, impl Iterator<Item = (u32, ScriptBuf)> + Clone> {
-        self.indexed_graph.index.spks_of_all_keychains()
+        self.indexed_graph.index.all_unbounded_spk_iters()
     }
 
-    /// Gets an iterator over all the script pubkeys in a single keychain.
+    /// Get an unbounded script pubkey iterator for the given `keychain`.
     ///
-    /// See [`spks_of_all_keychains`] for more documentation
+    /// See [`all_unbounded_spk_iters`] for more documentation
     ///
-    /// [`spks_of_all_keychains`]: Self::spks_of_all_keychains
-    pub fn spks_of_keychain(
+    /// [`all_unbounded_spk_iters`]: Self::all_unbounded_spk_iters
+    pub fn unbounded_spk_iter(
         &self,
         keychain: KeychainKind,
     ) -> impl Iterator<Item = (u32, ScriptBuf)> + Clone {
-        self.indexed_graph.index.spks_of_keychain(&keychain)
+        self.indexed_graph.index.unbounded_spk_iter(&keychain)
     }
 
     /// Returns the utxo owned by this wallet corresponding to `outpoint` if it exists in the
     /// wallet's database.
     pub fn get_utxo(&self, op: OutPoint) -> Option<LocalOutput> {
-        let (&spk_i, _) = self.indexed_graph.index.txout(op)?;
+        let (keychain, index, _) = self.indexed_graph.index.txout(op)?;
         self.indexed_graph
             .graph()
             .filter_chain_unspents(
                 &self.chain,
                 self.chain.tip().block_id(),
-                core::iter::once((spk_i, op)),
+                core::iter::once(((), op)),
             )
-            .map(|((k, i), full_txo)| new_local_utxo(k, i, full_txo))
+            .map(|(_, full_txo)| new_local_utxo(keychain, index, full_txo))
             .next()
     }
 
@@ -1445,7 +1486,7 @@ impl<D> Wallet<D> {
                 let ((index, spk), index_changeset) =
                     self.indexed_graph.index.next_unused_spk(&change_keychain);
                 let spk = spk.into();
-                self.indexed_graph.index.mark_used(&change_keychain, index);
+                self.indexed_graph.index.mark_used(change_keychain, index);
                 self.persist
                     .stage(ChangeSet::from(indexed_tx_graph::ChangeSet::from(
                         index_changeset,
@@ -1626,7 +1667,7 @@ impl<D> Wallet<D> {
                     .into();
 
                 let weighted_utxo = match txout_index.index_of_spk(&txout.script_pubkey) {
-                    Some(&(keychain, derivation_index)) => {
+                    Some((keychain, derivation_index)) => {
                         #[allow(deprecated)]
                         let satisfaction_weight = self
                             .get_descriptor_for_keychain(keychain)
@@ -1670,7 +1711,7 @@ impl<D> Wallet<D> {
             for (index, txout) in tx.output.iter().enumerate() {
                 let change_type = self.map_keychain(KeychainKind::Internal);
                 match txout_index.index_of_spk(&txout.script_pubkey) {
-                    Some(&(keychain, _)) if keychain == change_type => change_index = Some(index),
+                    Some((keychain, _)) if keychain == change_type => change_index = Some(index),
                     _ => {}
                 }
             }
@@ -1925,10 +1966,10 @@ impl<D> Wallet<D> {
     pub fn cancel_tx(&mut self, tx: &Transaction) {
         let txout_index = &mut self.indexed_graph.index;
         for txout in &tx.output {
-            if let Some(&(keychain, index)) = txout_index.index_of_spk(&txout.script_pubkey) {
+            if let Some((keychain, index)) = txout_index.index_of_spk(&txout.script_pubkey) {
                 // NOTE: unmark_used will **not** make something unused if it has actually been used
                 // by a tx in the tracker. It only removes the superficial marking.
-                txout_index.unmark_used(&keychain, index);
+                txout_index.unmark_used(keychain, index);
             }
         }
     }
@@ -1944,7 +1985,7 @@ impl<D> Wallet<D> {
     }
 
     fn get_descriptor_for_txout(&self, txout: &TxOut) -> Option<DerivedDescriptor> {
-        let &(keychain, child) = self
+        let (keychain, child) = self
             .indexed_graph
             .index
             .index_of_spk(&txout.script_pubkey)?;
@@ -2158,7 +2199,7 @@ impl<D> Wallet<D> {
     {
         // Try to find the prev_script in our db to figure out if this is internal or external,
         // and the derivation index
-        let &(keychain, child) = self
+        let (keychain, child) = self
             .indexed_graph
             .index
             .index_of_spk(&utxo.txout.script_pubkey)
@@ -2212,7 +2253,7 @@ impl<D> Wallet<D> {
 
         // Try to figure out the keychain and derivation for every input and output
         for (is_input, index, out) in utxos.into_iter() {
-            if let Some(&(keychain, child)) =
+            if let Some((keychain, child)) =
                 self.indexed_graph.index.index_of_spk(&out.script_pubkey)
             {
                 let desc = self.get_descriptor_for_keychain(keychain);

crates/bdk/src/wallet/signer.rs

@@ -80,6 +80,7 @@
 //! ```
 
 use crate::collections::BTreeMap;
+use alloc::string::String;
 use alloc::sync::Arc;
 use alloc::vec::Vec;
 use core::cmp::Ordering;
@@ -162,16 +163,10 @@ pub enum SignerError {
     SighashError(sighash::Error),
     /// Miniscript PSBT error
     MiniscriptPsbt(MiniscriptPsbtError),
-    /// Error while signing using hardware wallets
-    #[cfg(feature = "hardware-signer")]
-    HWIError(hwi::error::Error),
-}
-
-#[cfg(feature = "hardware-signer")]
-impl From<hwi::error::Error> for SignerError {
-    fn from(e: hwi::error::Error) -> Self {
-        SignerError::HWIError(e)
-    }
+    /// To be used only by external libraries implementing [`InputSigner`] or
+    /// [`TransactionSigner`], so that they can return their own custom errors, without having to
+    /// modify [`SignerError`] in BDK.
+    External(String),
 }
 
 impl From<sighash::Error> for SignerError {
@@ -196,8 +191,7 @@ impl fmt::Display for SignerError {
             Self::InvalidSighash => write!(f, "Invalid SIGHASH for the signing context in use"),
             Self::SighashError(err) => write!(f, "Error while computing the hash to sign: {}", err),
             Self::MiniscriptPsbt(err) => write!(f, "Miniscript PSBT error: {}", err),
-            #[cfg(feature = "hardware-signer")]
-            Self::HWIError(err) => write!(f, "Error while signing using hardware wallets: {}", err),
+            Self::External(err) => write!(f, "{}", err),
         }
     }
 }
@@ -812,9 +806,10 @@ pub struct SignOptions {
 }
 
 /// Customize which taproot script-path leaves the signer should sign.
-#[derive(Debug, Clone, PartialEq, Eq)]
+#[derive(Default, Debug, Clone, PartialEq, Eq)]
 pub enum TapLeavesOptions {
     /// The signer will sign all the leaves it has a key for.
+    #[default]
     All,
     /// The signer won't sign leaves other than the ones specified. Note that it could still ignore
     /// some of the specified leaves, if it doesn't have the right key to sign them.
@@ -825,12 +820,6 @@ pub enum TapLeavesOptions {
     None,
 }
 
-impl Default for TapLeavesOptions {
-    fn default() -> Self {
-        TapLeavesOptions::All
-    }
-}
-
 #[allow(clippy::derivable_impls)]
 impl Default for SignOptions {
     fn default() -> Self {

crates/bdk/src/wallet/tx_builder.rs

@@ -811,9 +811,10 @@ impl<'a, D> TxBuilder<'a, D, DefaultCoinSelectionAlgorithm, BumpFee> {
 }
 
 /// Ordering of the transaction's inputs and outputs
-#[derive(Debug, Ord, PartialOrd, Eq, PartialEq, Hash, Clone, Copy)]
+#[derive(Default, Debug, Ord, PartialOrd, Eq, PartialEq, Hash, Clone, Copy)]
 pub enum TxOrdering {
     /// Randomized (default)
+    #[default]
     Shuffle,
     /// Unchanged
     Untouched,
@@ -821,12 +822,6 @@ pub enum TxOrdering {
     Bip69Lexicographic,
 }
 
-impl Default for TxOrdering {
-    fn default() -> Self {
-        TxOrdering::Shuffle
-    }
-}
-
 impl TxOrdering {
     /// Sort transaction inputs and outputs by [`TxOrdering`] variant
     pub fn sort_tx(&self, tx: &mut Transaction) {
@@ -880,9 +875,10 @@ impl RbfValue {
 }
 
 /// Policy regarding the use of change outputs when creating a transaction
-#[derive(Debug, Ord, PartialOrd, Eq, PartialEq, Hash, Clone, Copy)]
+#[derive(Default, Debug, Ord, PartialOrd, Eq, PartialEq, Hash, Clone, Copy)]
 pub enum ChangeSpendPolicy {
     /// Use both change and non-change outputs (default)
+    #[default]
     ChangeAllowed,
     /// Only use change outputs (see [`TxBuilder::only_spend_change`])
     OnlyChange,
@@ -890,12 +886,6 @@ pub enum ChangeSpendPolicy {
     ChangeForbidden,
 }
 
-impl Default for ChangeSpendPolicy {
-    fn default() -> Self {
-        ChangeSpendPolicy::ChangeAllowed
-    }
-}
-
 impl ChangeSpendPolicy {
     pub(crate) fn is_satisfied_by(&self, utxo: &LocalOutput) -> bool {
         match self {

crates/bdk/tests/wallet.rs

@@ -7,8 +7,8 @@ use bdk::signer::{SignOptions, SignerError};
 use bdk::wallet::coin_selection::{self, LargestFirstCoinSelection};
 use bdk::wallet::error::CreateTxError;
 use bdk::wallet::tx_builder::AddForeignUtxoError;
-use bdk::wallet::AddressIndex::*;
 use bdk::wallet::{AddressIndex, AddressInfo, Balance, Wallet};
+use bdk::wallet::{AddressIndex::*, NewError};
 use bdk::{FeeRate, KeychainKind};
 use bdk_chain::COINBASE_MATURITY;
 use bdk_chain::{BlockId, ConfirmationTime};
@@ -71,19 +71,33 @@ fn load_recovers_wallet() {
     let file_path = temp_dir.path().join("store.db");
 
     // create new wallet
-    let wallet_keychains = {
+    let wallet_spk_index = {
         let db = bdk_file_store::Store::create_new(DB_MAGIC, &file_path).expect("must create db");
-        let wallet =
-            Wallet::new(get_test_wpkh(), None, db, Network::Testnet).expect("must init wallet");
-        wallet.keychains().clone()
+        let mut wallet = Wallet::new(get_test_tr_single_sig_xprv(), None, db, Network::Testnet)
+            .expect("must init wallet");
+
+        wallet.try_get_address(New).unwrap();
+        wallet.spk_index().clone()
     };
 
     // recover wallet
     {
         let db = bdk_file_store::Store::open(DB_MAGIC, &file_path).expect("must recover db");
-        let wallet = Wallet::load(get_test_wpkh(), None, db).expect("must recover wallet");
+        let wallet =
+            Wallet::load(get_test_tr_single_sig_xprv(), None, db).expect("must recover wallet");
         assert_eq!(wallet.network(), Network::Testnet);
-        assert_eq!(wallet.spk_index().keychains(), &wallet_keychains);
+        assert_eq!(wallet.spk_index().keychains(), wallet_spk_index.keychains());
+        assert_eq!(
+            wallet.spk_index().last_revealed_indices(),
+            wallet_spk_index.last_revealed_indices()
+        );
+    }
+
+    // `new` can only be called on empty db
+    {
+        let db = bdk_file_store::Store::open(DB_MAGIC, &file_path).expect("must recover db");
+        let result = Wallet::new(get_test_tr_single_sig_xprv(), None, db, Network::Testnet);
+        assert!(matches!(result, Err(NewError::NonEmptyDatabase)));
     }
 }
 
@@ -92,7 +106,7 @@ fn new_or_load() {
     let temp_dir = tempfile::tempdir().expect("must create tempdir");
     let file_path = temp_dir.path().join("store.db");
 
-    // init wallet when non-existant
+    // init wallet when non-existent
     let wallet_keychains = {
         let db = bdk_file_store::Store::open_or_create_new(DB_MAGIC, &file_path)
             .expect("must create db");
@@ -3577,41 +3591,6 @@ fn test_fee_rate_sign_grinding_low_r() {
     assert_fee_rate!(psbt, fee.unwrap_or(0), fee_rate);
 }
 
-// #[cfg(feature = "test-hardware-signer")]
-// #[test]
-// fn test_hardware_signer() {
-//     use std::sync::Arc;
-//
-//     use bdk::signer::SignerOrdering;
-//     use bdk::wallet::hardwaresigner::HWISigner;
-//     use hwi::types::HWIChain;
-//     use hwi::HWIClient;
-//
-//     let mut devices = HWIClient::enumerate().unwrap();
-//     if devices.is_empty() {
-//         panic!("No devices found!");
-//     }
-//     let device = devices.remove(0).unwrap();
-//     let client = HWIClient::get_client(&device, true, HWIChain::Regtest).unwrap();
-//     let descriptors = client.get_descriptors::<String>(None).unwrap();
-//     let custom_signer = HWISigner::from_device(&device, HWIChain::Regtest).unwrap();
-//
-//     let (mut wallet, _) = get_funded_wallet(&descriptors.internal[0]);
-//     wallet.add_signer(
-//         KeychainKind::External,
-//         SignerOrdering(200),
-//         Arc::new(custom_signer),
-//     );
-//
-//     let addr = wallet.get_address(LastUnused);
-//     let mut builder = wallet.build_tx();
-//     builder.drain_to(addr.script_pubkey()).drain_wallet();
-//     let (mut psbt, _) = builder.finish().unwrap();
-//
-//     let finalized = wallet.sign(&mut psbt, Default::default()).unwrap();
-//     assert!(finalized);
-// }
-
 #[test]
 fn test_taproot_load_descriptor_duplicated_keys() {
     // Added after issue https://github.com/bitcoindevkit/bdk/issues/760

crates/bitcoind_rpc/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "bdk_bitcoind_rpc"
-version = "0.1.0"
+version = "0.3.0"
 edition = "2021"
 rust-version = "1.63"
 homepage = "https://bitcoindevkit.org"
@@ -16,7 +16,7 @@ readme = "README.md"
 # For no-std, remember to enable the bitcoin/no-std feature
 bitcoin = { version = "0.30", default-features = false }
 bitcoincore-rpc = { version = "0.17" }
-bdk_chain = { path = "../chain", version = "0.6", default-features = false }
+bdk_chain = { path = "../chain", version = "0.8", default-features = false }
 
 [dev-dependencies]
 bitcoind = { version = "0.33", features = ["25_0"] }

crates/chain/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "bdk_chain"
-version = "0.6.0"
+version = "0.8.0"
 edition = "2021"
 rust-version = "1.63"
 homepage = "https://bitcoindevkit.org"

crates/chain/src/keychain.rs

@@ -58,8 +58,9 @@ impl<K: Ord> Append for ChangeSet<K> {
                 *index = other_index.max(*index);
             }
         });
-
-        self.0.append(&mut other.0);
+        // We use `extend` instead of `BTreeMap::append` due to performance issues with `append`.
+        // Refer to https://github.com/rust-lang/rust/issues/34666#issuecomment-675658420
+        self.0.extend(other.0);
     }
 
     /// Returns whether the changeset are empty.

crates/chain/src/keychain/txout_index.rs

@@ -5,23 +5,56 @@ use crate::{
     spk_iter::BIP32_MAX_INDEX,
     SpkIterator, SpkTxOutIndex,
 };
-use alloc::vec::Vec;
-use bitcoin::{OutPoint, Script, TxOut};
-use core::{fmt::Debug, ops::Deref};
+use bitcoin::{OutPoint, Script, Transaction, TxOut, Txid};
+use core::{
+    fmt::Debug,
+    ops::{Bound, RangeBounds},
+};
 
 use crate::Append;
 
-/// A convenient wrapper around [`SpkTxOutIndex`] that relates script pubkeys to miniscript public
-/// [`Descriptor`]s.
+const DEFAULT_LOOKAHEAD: u32 = 1_000;
+
+/// [`KeychainTxOutIndex`] controls how script pubkeys are revealed for multiple keychains, and
+/// indexes [`TxOut`]s with them.
 ///
-/// Descriptors are referenced by the provided keychain generic (`K`).
+/// A single keychain is a chain of script pubkeys derived from a single [`Descriptor`]. Keychains
+/// are identified using the `K` generic. Script pubkeys are identified by the keychain that they
+/// are derived from `K`, as well as the derivation index `u32`.
 ///
-/// Script pubkeys for a descriptor are revealed chronologically from index 0. I.e., If the last
-/// revealed index of a descriptor is 5; scripts of indices 0 to 4 are guaranteed to be already
-/// revealed. In addition to revealed scripts, we have a `lookahead` parameter for each keychain,
-/// which defines the number of script pubkeys to store ahead of the last revealed index.
+/// # Revealed script pubkeys
 ///
-/// Methods that could update the last revealed index will return [`super::ChangeSet`] to report
+/// Tracking how script pubkeys are revealed is useful for collecting chain data. For example, if
+/// the user has requested 5 script pubkeys (to receive money with), we only need to use those
+/// script pubkeys to scan for chain data.
+///
+/// Call [`reveal_to_target`] or [`reveal_next_spk`] to reveal more script pubkeys.
+/// Call [`revealed_keychain_spks`] or [`revealed_spks`] to iterate through revealed script pubkeys.
+///
+/// # Lookahead script pubkeys
+///
+/// When an user first recovers a wallet (i.e. from a recovery phrase and/or descriptor), we will
+/// NOT have knowledge of which script pubkeys are revealed. So when we index a transaction or
+/// txout (using [`index_tx`]/[`index_txout`]) we scan the txouts against script pubkeys derived
+/// above the last revealed index. These additionally-derived script pubkeys are called the
+/// lookahead.
+///
+/// The [`KeychainTxOutIndex`] is constructed with the `lookahead` and cannot be altered. The
+/// default `lookahead` count is 1000. Use [`new`] to set a custom `lookahead`.
+///
+/// # Unbounded script pubkey iterator
+///
+/// For script-pubkey-based chain sources (such as Electrum/Esplora), an initial scan is best done
+/// by iterating though derived script pubkeys one by one and requesting transaction histories for
+/// each script pubkey. We will stop after x-number of script pubkeys have empty histories. An
+/// unbounded script pubkey iterator is useful to pass to such a chain source.
+///
+/// Call [`unbounded_spk_iter`] to get an unbounded script pubkey iterator for a given keychain.
+/// Call [`all_unbounded_spk_iters`] to get unbounded script pubkey iterators for all keychains.
+///
+/// # Change sets
+///
+/// Methods that can update the last revealed index will return [`super::ChangeSet`] to report
 /// these changes. This can be persisted for future recovery.
 ///
 /// ## Synopsis
@@ -46,7 +79,7 @@ use crate::Append;
 /// # let secp = bdk_chain::bitcoin::secp256k1::Secp256k1::signing_only();
 /// # let (external_descriptor,_) = Descriptor::<DescriptorPublicKey>::parse_descriptor(&secp, "tr([73c5da0a/86'/0'/0']xprv9xgqHN7yz9MwCkxsBPN5qetuNdQSUttZNKw1dcYTV4mkaAFiBVGQziHs3NRSWMkCzvgjEe3n9xV8oYywvM8at9yRqyaZVz6TYYhX98VjsUk/0/*)").unwrap();
 /// # let (internal_descriptor,_) = Descriptor::<DescriptorPublicKey>::parse_descriptor(&secp, "tr([73c5da0a/86'/0'/0']xprv9xgqHN7yz9MwCkxsBPN5qetuNdQSUttZNKw1dcYTV4mkaAFiBVGQziHs3NRSWMkCzvgjEe3n9xV8oYywvM8at9yRqyaZVz6TYYhX98VjsUk/1/*)").unwrap();
-/// # let descriptor_for_user_42 = external_descriptor.clone();
+/// # let (descriptor_for_user_42, _) = Descriptor::<DescriptorPublicKey>::parse_descriptor(&secp, "tr([73c5da0a/86'/0'/0']xprv9xgqHN7yz9MwCkxsBPN5qetuNdQSUttZNKw1dcYTV4mkaAFiBVGQziHs3NRSWMkCzvgjEe3n9xV8oYywvM8at9yRqyaZVz6TYYhX98VjsUk/2/*)").unwrap();
 /// txout_index.add_keychain(MyKeychain::External, external_descriptor);
 /// txout_index.add_keychain(MyKeychain::Internal, internal_descriptor);
 /// txout_index.add_keychain(MyKeychain::MyAppUser { user_id: 42 }, descriptor_for_user_42);
@@ -57,6 +90,15 @@ use crate::Append;
 /// [`Ord`]: core::cmp::Ord
 /// [`SpkTxOutIndex`]: crate::spk_txout_index::SpkTxOutIndex
 /// [`Descriptor`]: crate::miniscript::Descriptor
+/// [`reveal_to_target`]: KeychainTxOutIndex::reveal_to_target
+/// [`reveal_next_spk`]: KeychainTxOutIndex::reveal_next_spk
+/// [`revealed_keychain_spks`]: KeychainTxOutIndex::revealed_keychain_spks
+/// [`revealed_spks`]: KeychainTxOutIndex::revealed_spks
+/// [`index_tx`]: KeychainTxOutIndex::index_tx
+/// [`index_txout`]: KeychainTxOutIndex::index_txout
+/// [`new`]: KeychainTxOutIndex::new
+/// [`unbounded_spk_iter`]: KeychainTxOutIndex::unbounded_spk_iter
+/// [`all_unbounded_spk_iters`]: KeychainTxOutIndex::all_unbounded_spk_iters
 #[derive(Clone, Debug)]
 pub struct KeychainTxOutIndex<K> {
     inner: SpkTxOutIndex<(K, u32)>,
@@ -65,25 +107,12 @@ pub struct KeychainTxOutIndex<K> {
     // last revealed indexes
     last_revealed: BTreeMap<K, u32>,
     // lookahead settings for each keychain
-    lookahead: BTreeMap<K, u32>,
+    lookahead: u32,
 }
 
 impl<K> Default for KeychainTxOutIndex<K> {
     fn default() -> Self {
-        Self {
-            inner: SpkTxOutIndex::default(),
-            keychains: BTreeMap::default(),
-            last_revealed: BTreeMap::default(),
-            lookahead: BTreeMap::default(),
-        }
-    }
-}
-
-impl<K> Deref for KeychainTxOutIndex<K> {
-    type Target = SpkTxOutIndex<(K, u32)>;
-
-    fn deref(&self) -> &Self::Target {
-        &self.inner
+        Self::new(DEFAULT_LOOKAHEAD)
     }
 }
 
@@ -114,12 +143,37 @@ impl<K: Clone + Ord + Debug> Indexer for KeychainTxOutIndex<K> {
     }
 
     fn is_tx_relevant(&self, tx: &bitcoin::Transaction) -> bool {
-        self.is_relevant(tx)
+        self.inner.is_relevant(tx)
     }
 }
 
+impl<K> KeychainTxOutIndex<K> {
+    /// Construct a [`KeychainTxOutIndex`] with the given `lookahead`.
+    ///
+    /// The `lookahead` is the number of script pubkeys to derive and cache from the internal
+    /// descriptors over and above the last revealed script index. Without a lookahead the index
+    /// will miss outputs you own when processing transactions whose output script pubkeys lie
+    /// beyond the last revealed index. In certain situations, such as when performing an initial
+    /// scan of the blockchain during wallet import, it may be uncertain or unknown what the index
+    /// of the last revealed script pubkey actually is.
+    ///
+    /// Refer to [struct-level docs](KeychainTxOutIndex) for more about `lookahead`.
+    pub fn new(lookahead: u32) -> Self {
+        Self {
+            inner: SpkTxOutIndex::default(),
+            keychains: BTreeMap::new(),
+            last_revealed: BTreeMap::new(),
+            lookahead,
+        }
+    }
+}
+
+/// Methods that are *re-exposed* from the internal [`SpkTxOutIndex`].
 impl<K: Clone + Ord + Debug> KeychainTxOutIndex<K> {
     /// Return a reference to the internal [`SpkTxOutIndex`].
+    ///
+    /// **WARNING:** The internal index will contain lookahead spks. Refer to
+    /// [struct-level docs](KeychainTxOutIndex) for more about `lookahead`.
     pub fn inner(&self) -> &SpkTxOutIndex<(K, u32)> {
         &self.inner
     }
@@ -129,7 +183,116 @@ impl<K: Clone + Ord + Debug> KeychainTxOutIndex<K> {
         self.inner.outpoints()
     }
 
-    /// Return a reference to the internal map of the keychain to descriptors.
+    /// Iterate over known txouts that spend to tracked script pubkeys.
+    pub fn txouts(
+        &self,
+    ) -> impl DoubleEndedIterator<Item = (K, u32, OutPoint, &TxOut)> + ExactSizeIterator {
+        self.inner
+            .txouts()
+            .map(|((k, i), op, txo)| (k.clone(), *i, op, txo))
+    }
+
+    /// Finds all txouts on a transaction that has previously been scanned and indexed.
+    pub fn txouts_in_tx(
+        &self,
+        txid: Txid,
+    ) -> impl DoubleEndedIterator<Item = (K, u32, OutPoint, &TxOut)> {
+        self.inner
+            .txouts_in_tx(txid)
+            .map(|((k, i), op, txo)| (k.clone(), *i, op, txo))
+    }
+
+    /// Return the [`TxOut`] of `outpoint` if it has been indexed.
+    ///
+    /// The associated keychain and keychain index of the txout's spk is also returned.
+    ///
+    /// This calls [`SpkTxOutIndex::txout`] internally.
+    pub fn txout(&self, outpoint: OutPoint) -> Option<(K, u32, &TxOut)> {
+        self.inner
+            .txout(outpoint)
+            .map(|((k, i), txo)| (k.clone(), *i, txo))
+    }
+
+    /// Return the script that exists under the given `keychain`'s `index`.
+    ///
+    /// This calls [`SpkTxOutIndex::spk_at_index`] internally.
+    pub fn spk_at_index(&self, keychain: K, index: u32) -> Option<&Script> {
+        self.inner.spk_at_index(&(keychain, index))
+    }
+
+    /// Returns the keychain and keychain index associated with the spk.
+    ///
+    /// This calls [`SpkTxOutIndex::index_of_spk`] internally.
+    pub fn index_of_spk(&self, script: &Script) -> Option<(K, u32)> {
+        self.inner.index_of_spk(script).cloned()
+    }
+
+    /// Returns whether the spk under the `keychain`'s `index` has been used.
+    ///
+    /// Here, "unused" means that after the script pubkey was stored in the index, the index has
+    /// never scanned a transaction output with it.
+    ///
+    /// This calls [`SpkTxOutIndex::is_used`] internally.
+    pub fn is_used(&self, keychain: K, index: u32) -> bool {
+        self.inner.is_used(&(keychain, index))
+    }
+
+    /// Marks the script pubkey at `index` as used even though the tracker hasn't seen an output
+    /// with it.
+    ///
+    /// This only has an effect when the `index` had been added to `self` already and was unused.
+    ///
+    /// Returns whether the `index` was initially present as `unused`.
+    ///
+    /// This is useful when you want to reserve a script pubkey for something but don't want to add
+    /// the transaction output using it to the index yet. Other callers will consider `index` on
+    /// `keychain` used until you call [`unmark_used`].
+    ///
+    /// This calls [`SpkTxOutIndex::mark_used`] internally.
+    ///
+    /// [`unmark_used`]: Self::unmark_used
+    pub fn mark_used(&mut self, keychain: K, index: u32) -> bool {
+        self.inner.mark_used(&(keychain, index))
+    }
+
+    /// Undoes the effect of [`mark_used`]. Returns whether the `index` is inserted back into
+    /// `unused`.
+    ///
+    /// Note that if `self` has scanned an output with this script pubkey, then this will have no
+    /// effect.
+    ///
+    /// This calls [`SpkTxOutIndex::unmark_used`] internally.
+    ///
+    /// [`mark_used`]: Self::mark_used
+    pub fn unmark_used(&mut self, keychain: K, index: u32) -> bool {
+        self.inner.unmark_used(&(keychain, index))
+    }
+
+    /// Computes total input value going from script pubkeys in the index (sent) and the total output
+    /// value going to script pubkeys in the index (received) in `tx`. For the `sent` to be computed
+    /// correctly, the output being spent must have already been scanned by the index. Calculating
+    /// received just uses the [`Transaction`] outputs directly, so it will be correct even if it has
+    /// not been scanned.
+    ///
+    /// This calls [`SpkTxOutIndex::sent_and_received`] internally.
+    pub fn sent_and_received(&self, tx: &Transaction) -> (u64, u64) {
+        self.inner.sent_and_received(tx)
+    }
+
+    /// Computes the net value that this transaction gives to the script pubkeys in the index and
+    /// *takes* from the transaction outputs in the index. Shorthand for calling
+    /// [`sent_and_received`] and subtracting sent from received.
+    ///
+    /// This calls [`SpkTxOutIndex::net_value`] internally.
+    ///
+    /// [`sent_and_received`]: Self::sent_and_received
+    pub fn net_value(&self, tx: &Transaction) -> i64 {
+        self.inner.net_value(tx)
+    }
+}
+
+impl<K: Clone + Ord + Debug> KeychainTxOutIndex<K> {
+    /// Return a reference to the internal map of keychain to descriptors.
     pub fn keychains(&self) -> &BTreeMap<K, Descriptor<DescriptorPublicKey>> {
         &self.keychains
     }
@@ -145,54 +308,22 @@ impl<K: Clone + Ord + Debug> KeychainTxOutIndex<K> {
     pub fn add_keychain(&mut self, keychain: K, descriptor: Descriptor<DescriptorPublicKey>) {
         let old_descriptor = &*self
             .keychains
-            .entry(keychain)
+            .entry(keychain.clone())
             .or_insert_with(|| descriptor.clone());
         assert_eq!(
             &descriptor, old_descriptor,
             "keychain already contains a different descriptor"
         );
+        self.replenish_lookahead(&keychain, self.lookahead);
     }
 
-    /// Return the lookahead setting for each keychain.
+    /// Get the lookahead setting.
     ///
-    /// Refer to [`set_lookahead`] for a deeper explanation of the `lookahead`.
+    /// Refer to [`new`] for more information on the `lookahead`.
     ///
-    /// [`set_lookahead`]: Self::set_lookahead
-    pub fn lookaheads(&self) -> &BTreeMap<K, u32> {
-        &self.lookahead
-    }
-
-    /// Convenience method to call [`set_lookahead`] for all keychains.
-    ///
-    /// [`set_lookahead`]: Self::set_lookahead
-    pub fn set_lookahead_for_all(&mut self, lookahead: u32) {
-        for keychain in &self.keychains.keys().cloned().collect::<Vec<_>>() {
-            self.set_lookahead(keychain, lookahead);
-        }
-    }
-
-    /// Set the lookahead count for `keychain`.
-    ///
-    /// The lookahead is the number of scripts to cache ahead of the last revealed script index. This
-    /// is useful to find outputs you own when processing block data that lie beyond the last revealed
-    /// index. In certain situations, such as when performing an initial scan of the blockchain during
-    /// wallet import, it may be uncertain or unknown what the last revealed index is.
-    ///
-    /// # Panics
-    ///
-    /// This will panic if the `keychain` does not exist.
-    pub fn set_lookahead(&mut self, keychain: &K, lookahead: u32) {
-        self.lookahead.insert(keychain.clone(), lookahead);
-        self.replenish_lookahead(keychain);
-    }
-
-    /// Convenience method to call [`lookahead_to_target`] for multiple keychains.
-    ///
-    /// [`lookahead_to_target`]: Self::lookahead_to_target
-    pub fn lookahead_to_target_multi(&mut self, target_indexes: BTreeMap<K, u32>) {
-        for (keychain, target_index) in target_indexes {
-            self.lookahead_to_target(&keychain, target_index)
-        }
+    /// [`new`]: Self::new
+    pub fn lookahead(&self) -> u32 {
+        self.lookahead
     }
 
     /// Store lookahead scripts until `target_index`.
@@ -201,22 +332,14 @@ impl<K: Clone + Ord + Debug> KeychainTxOutIndex<K> {
     pub fn lookahead_to_target(&mut self, keychain: &K, target_index: u32) {
         let next_index = self.next_store_index(keychain);
         if let Some(temp_lookahead) = target_index.checked_sub(next_index).filter(|&v| v > 0) {
-            let old_lookahead = self.lookahead.insert(keychain.clone(), temp_lookahead);
-            self.replenish_lookahead(keychain);
-
-            // revert
-            match old_lookahead {
-                Some(lookahead) => self.lookahead.insert(keychain.clone(), lookahead),
-                None => self.lookahead.remove(keychain),
-            };
+            self.replenish_lookahead(keychain, temp_lookahead);
         }
     }
 
-    fn replenish_lookahead(&mut self, keychain: &K) {
+    fn replenish_lookahead(&mut self, keychain: &K, lookahead: u32) {
         let descriptor = self.keychains.get(keychain).expect("keychain must exist");
         let next_store_index = self.next_store_index(keychain);
         let next_reveal_index = self.last_revealed.get(keychain).map_or(0, |v| *v + 1);
-        let lookahead = self.lookahead.get(keychain).map_or(0, |v| *v);
 
         for (new_index, new_spk) in
             SpkIterator::new_with_range(descriptor, next_store_index..next_reveal_index + lookahead)
@@ -231,64 +354,74 @@ impl<K: Clone + Ord + Debug> KeychainTxOutIndex<K> {
     fn next_store_index(&self, keychain: &K) -> u32 {
         self.inner()
             .all_spks()
+            // This range is filtering out the spks with a keychain different than
+            // `keychain`. We don't use filter here as range is more optimized.
             .range((keychain.clone(), u32::MIN)..(keychain.clone(), u32::MAX))
             .last()
-            .map_or(0, |((_, v), _)| *v + 1)
+            .map_or(0, |((_, index), _)| *index + 1)
     }
 
-    /// Generates script pubkey iterators for every `keychain`. The iterators iterate over all
-    /// derivable script pubkeys.
-    pub fn spks_of_all_keychains(
+    /// Get an unbounded spk iterator over a given `keychain`.
+    ///
+    /// # Panics
+    ///
+    /// This will panic if the given `keychain`'s descriptor does not exist.
+    pub fn unbounded_spk_iter(&self, keychain: &K) -> SpkIterator<Descriptor<DescriptorPublicKey>> {
+        SpkIterator::new(
+            self.keychains
+                .get(keychain)
+                .expect("keychain does not exist")
+                .clone(),
+        )
+    }
+
+    /// Get unbounded spk iterators for all keychains.
+    pub fn all_unbounded_spk_iters(
         &self,
     ) -> BTreeMap<K, SpkIterator<Descriptor<DescriptorPublicKey>>> {
         self.keychains
             .iter()
-            .map(|(keychain, descriptor)| {
-                (
-                    keychain.clone(),
-                    SpkIterator::new_with_range(descriptor.clone(), 0..),
-                )
-            })
+            .map(|(k, descriptor)| (k.clone(), SpkIterator::new(descriptor.clone())))
             .collect()
     }
 
-    /// Generates a script pubkey iterator for the given `keychain`'s descriptor (if it exists). The
-    /// iterator iterates over all derivable scripts of the keychain's descriptor.
-    ///
-    /// # Panics
-    ///
-    /// This will panic if the `keychain` does not exist.
-    pub fn spks_of_keychain(&self, keychain: &K) -> SpkIterator<Descriptor<DescriptorPublicKey>> {
-        let descriptor = self
-            .keychains
-            .get(keychain)
-            .expect("keychain must exist")
-            .clone();
-        SpkIterator::new_with_range(descriptor, 0..)
+    /// Iterate over revealed spks of all keychains.
+    pub fn revealed_spks(&self) -> impl DoubleEndedIterator<Item = (K, u32, &Script)> + Clone {
+        self.keychains.keys().flat_map(|keychain| {
+            self.revealed_keychain_spks(keychain)
+                .map(|(i, spk)| (keychain.clone(), i, spk))
+        })
     }
 
-    /// Convenience method to get [`revealed_spks_of_keychain`] of all keychains.
-    ///
-    /// [`revealed_spks_of_keychain`]: Self::revealed_spks_of_keychain
-    pub fn revealed_spks_of_all_keychains(
-        &self,
-    ) -> BTreeMap<K, impl Iterator<Item = (u32, &Script)> + Clone> {
-        self.keychains
-            .keys()
-            .map(|keychain| (keychain.clone(), self.revealed_spks_of_keychain(keychain)))
-            .collect()
-    }
-
-    /// Iterates over the script pubkeys revealed by this index under `keychain`.
-    pub fn revealed_spks_of_keychain(
+    /// Iterate over revealed spks of the given `keychain`.
+    pub fn revealed_keychain_spks(
         &self,
         keychain: &K,
     ) -> impl DoubleEndedIterator<Item = (u32, &Script)> + Clone {
-        let next_index = self.last_revealed.get(keychain).map_or(0, |v| *v + 1);
+        let next_i = self.last_revealed.get(keychain).map_or(0, |&i| i + 1);
         self.inner
             .all_spks()
-            .range((keychain.clone(), u32::MIN)..(keychain.clone(), next_index))
-            .map(|((_, derivation_index), spk)| (*derivation_index, spk.as_script()))
+            .range((keychain.clone(), u32::MIN)..(keychain.clone(), next_i))
+            .map(|((_, i), spk)| (*i, spk.as_script()))
+    }
+
+    /// Iterate over revealed, but unused, spks of all keychains.
+    pub fn unused_spks(&self) -> impl DoubleEndedIterator<Item = (K, u32, &Script)> + Clone {
+        self.keychains.keys().flat_map(|keychain| {
+            self.unused_keychain_spks(keychain)
+                .map(|(i, spk)| (keychain.clone(), i, spk))
+        })
+    }
+
+    /// Iterate over revealed, but unused, spks of the given `keychain`.
+    pub fn unused_keychain_spks(
+        &self,
+        keychain: &K,
+    ) -> impl DoubleEndedIterator<Item = (u32, &Script)> + Clone {
+        let next_i = self.last_revealed.get(keychain).map_or(0, |&i| i + 1);
+        self.inner
+            .unused_spks((keychain.clone(), u32::MIN)..(keychain.clone(), next_i))
+            .map(|((_, i), spk)| (*i, spk))
     }
 
     /// Get the next derivation index for `keychain`. The next index is the index after the last revealed
@@ -387,55 +520,45 @@ impl<K: Clone + Ord + Debug> KeychainTxOutIndex<K> {
         let has_wildcard = descriptor.has_wildcard();
 
         let target_index = if has_wildcard { target_index } else { 0 };
-        let next_reveal_index = self.last_revealed.get(keychain).map_or(0, |v| *v + 1);
-        let lookahead = self.lookahead.get(keychain).map_or(0, |v| *v);
+        let next_reveal_index = self
+            .last_revealed
+            .get(keychain)
+            .map_or(0, |index| *index + 1);
 
-        debug_assert_eq!(
-            next_reveal_index + lookahead,
-            self.next_store_index(keychain)
-        );
+        debug_assert!(next_reveal_index + self.lookahead >= self.next_store_index(keychain));
 
-        // if we need to reveal new indices, the latest revealed index goes here
-        let mut reveal_to_index = None;
-
-        // if the target is not yet revealed, but is already stored (due to lookahead), we need to
-        // set the `reveal_to_index` as target here (as the `for` loop below only updates
-        // `reveal_to_index` for indexes that are NOT stored)
-        if next_reveal_index <= target_index && target_index < next_reveal_index + lookahead {
-            reveal_to_index = Some(target_index);
-        }
-
-        // we range over indexes that are not stored
-        let range = next_reveal_index + lookahead..=target_index + lookahead;
-        for (new_index, new_spk) in SpkIterator::new_with_range(descriptor, range) {
-            let _inserted = self
-                .inner
-                .insert_spk((keychain.clone(), new_index), new_spk);
-            debug_assert!(_inserted, "must not have existing spk",);
-
-            // everything after `target_index` is stored for lookahead only
-            if new_index <= target_index {
-                reveal_to_index = Some(new_index);
-            }
-        }
-
-        match reveal_to_index {
-            Some(index) => {
-                let _old_index = self.last_revealed.insert(keychain.clone(), index);
-                debug_assert!(_old_index < Some(index));
-                (
-                    SpkIterator::new_with_range(descriptor.clone(), next_reveal_index..index + 1),
-                    super::ChangeSet(core::iter::once((keychain.clone(), index)).collect()),
-                )
-            }
-            None => (
+        // If the target_index is already revealed, we are done
+        if next_reveal_index > target_index {
+            return (
                 SpkIterator::new_with_range(
                     descriptor.clone(),
                     next_reveal_index..next_reveal_index,
                 ),
                 super::ChangeSet::default(),
-            ),
+            );
         }
+
+        // We range over the indexes that are not stored and insert their spks in the index.
+        // Indexes from next_reveal_index to next_reveal_index + lookahead are already stored (due
+        // to lookahead), so we only range from next_reveal_index + lookahead to target + lookahead
+        let range = next_reveal_index + self.lookahead..=target_index + self.lookahead;
+        for (new_index, new_spk) in SpkIterator::new_with_range(descriptor, range) {
+            let _inserted = self
+                .inner
+                .insert_spk((keychain.clone(), new_index), new_spk);
+            debug_assert!(_inserted, "must not have existing spk");
+            debug_assert!(
+                has_wildcard || new_index == 0,
+                "non-wildcard descriptors must not iterate past index 0"
+            );
+        }
+
+        let _old_index = self.last_revealed.insert(keychain.clone(), target_index);
+        debug_assert!(_old_index < Some(target_index));
+        (
+            SpkIterator::new_with_range(descriptor.clone(), next_reveal_index..target_index + 1),
+            super::ChangeSet(core::iter::once((keychain.clone(), target_index)).collect()),
+        )
     }
 
     /// Attempts to reveal the next script pubkey for `keychain`.
@@ -475,13 +598,13 @@ impl<K: Clone + Ord + Debug> KeychainTxOutIndex<K> {
     ///
     /// Panics if `keychain` has never been added to the index
     pub fn next_unused_spk(&mut self, keychain: &K) -> ((u32, &Script), super::ChangeSet<K>) {
-        let need_new = self.unused_spks_of_keychain(keychain).next().is_none();
+        let need_new = self.unused_keychain_spks(keychain).next().is_none();
         // this rather strange branch is needed because of some lifetime issues
         if need_new {
             self.reveal_next_spk(keychain)
         } else {
             (
-                self.unused_spks_of_keychain(keychain)
+                self.unused_keychain_spks(keychain)
                     .next()
                     .expect("we already know next exists"),
                 super::ChangeSet::default(),
@@ -489,58 +612,44 @@ impl<K: Clone + Ord + Debug> KeychainTxOutIndex<K> {
         }
     }
 
-    /// Marks the script pubkey at `index` as used even though the tracker hasn't seen an output with it.
-    /// This only has an effect when the `index` had been added to `self` already and was unused.
-    ///
-    /// Returns whether the `index` was initially present as `unused`.
-    ///
-    /// This is useful when you want to reserve a script pubkey for something but don't want to add
-    /// the transaction output using it to the index yet. Other callers will consider `index` on
-    /// `keychain` used until you call [`unmark_used`].
-    ///
-    /// [`unmark_used`]: Self::unmark_used
-    pub fn mark_used(&mut self, keychain: &K, index: u32) -> bool {
-        self.inner.mark_used(&(keychain.clone(), index))
-    }
-
-    /// Undoes the effect of [`mark_used`]. Returns whether the `index` is inserted back into
-    /// `unused`.
-    ///
-    /// Note that if `self` has scanned an output with this script pubkey, then this will have no
-    /// effect.
-    ///
-    /// [`mark_used`]: Self::mark_used
-    pub fn unmark_used(&mut self, keychain: &K, index: u32) -> bool {
-        self.inner.unmark_used(&(keychain.clone(), index))
-    }
-
-    /// Iterates over all unused script pubkeys for a `keychain` stored in the index.
-    pub fn unused_spks_of_keychain(
-        &self,
-        keychain: &K,
-    ) -> impl DoubleEndedIterator<Item = (u32, &Script)> {
-        let next_index = self.last_revealed.get(keychain).map_or(0, |&v| v + 1);
-        let range = (keychain.clone(), u32::MIN)..(keychain.clone(), next_index);
-        self.inner
-            .unused_spks(range)
-            .map(|((_, i), script)| (*i, script))
-    }
-
-    /// Iterates over all the [`OutPoint`] that have a `TxOut` with a script pubkey derived from
+    /// Iterate over all [`OutPoint`]s that point to `TxOut`s with script pubkeys derived from
     /// `keychain`.
-    pub fn txouts_of_keychain(
+    ///
+    /// Use [`keychain_outpoints_in_range`](KeychainTxOutIndex::keychain_outpoints_in_range) to
+    /// iterate over a specific derivation range.
+    pub fn keychain_outpoints(
         &self,
         keychain: &K,
     ) -> impl DoubleEndedIterator<Item = (u32, OutPoint)> + '_ {
+        self.keychain_outpoints_in_range(keychain, ..)
+    }
+
+    /// Iterate over [`OutPoint`]s that point to `TxOut`s with script pubkeys derived from
+    /// `keychain` in a given derivation `range`.
+    pub fn keychain_outpoints_in_range(
+        &self,
+        keychain: &K,
+        range: impl RangeBounds<u32>,
+    ) -> impl DoubleEndedIterator<Item = (u32, OutPoint)> + '_ {
+        let start = match range.start_bound() {
+            Bound::Included(i) => Bound::Included((keychain.clone(), *i)),
+            Bound::Excluded(i) => Bound::Excluded((keychain.clone(), *i)),
+            Bound::Unbounded => Bound::Unbounded,
+        };
+        let end = match range.end_bound() {
+            Bound::Included(i) => Bound::Included((keychain.clone(), *i)),
+            Bound::Excluded(i) => Bound::Excluded((keychain.clone(), *i)),
+            Bound::Unbounded => Bound::Unbounded,
+        };
         self.inner
-            .outputs_in_range((keychain.clone(), u32::MIN)..(keychain.clone(), u32::MAX))
+            .outputs_in_range((start, end))
             .map(|((_, i), op)| (*i, op))
     }
 
     /// Returns the highest derivation index of the `keychain` where [`KeychainTxOutIndex`] has
     /// found a [`TxOut`] with it's script pubkey.
     pub fn last_used_index(&self, keychain: &K) -> Option<u32> {
-        self.txouts_of_keychain(keychain).last().map(|(i, _)| i)
+        self.keychain_outpoints(keychain).last().map(|(i, _)| i)
     }
 
     /// Returns the highest derivation index of each keychain that [`KeychainTxOutIndex`] has found

crates/chain/src/lib.rs

@@ -1,4 +1,4 @@
-//! This crate is a collection of core structures for [Bitcoin Dev Kit] (alpha release).
+//! This crate is a collection of core structures for [Bitcoin Dev Kit].
 //!
 //! The goal of this crate is to give wallets the mechanisms needed to:
 //!

crates/chain/src/local_chain.rs

@@ -420,6 +420,28 @@ impl LocalChain {
         Ok(changeset)
     }
 
+    /// Removes blocks from (and inclusive of) the given `block_id`.
+    ///
+    /// This will remove blocks with a height equal or greater than `block_id`, but only if
+    /// `block_id` exists in the chain.
+    ///
+    /// # Errors
+    ///
+    /// This will fail with [`MissingGenesisError`] if the caller attempts to disconnect from the
+    /// genesis block.
+    pub fn disconnect_from(&mut self, block_id: BlockId) -> Result<ChangeSet, MissingGenesisError> {
+        if self.index.get(&block_id.height) != Some(&block_id.hash) {
+            return Ok(ChangeSet::default());
+        }
+
+        let changeset = self
+            .index
+            .range(block_id.height..)
+            .map(|(&height, _)| (height, None))
+            .collect::<ChangeSet>();
+        self.apply_changeset(&changeset).map(|_| changeset)
+    }
+
     /// Reindex the heights in the chain from (and including) `from` height
     fn reindex(&mut self, from: u32) {
         let _ = self.index.split_off(&from);

crates/chain/src/spk_iter.rs

@@ -87,6 +87,11 @@ where
             secp: Secp256k1::verification_only(),
         }
     }
+
+    /// Get a reference to the internal descriptor.
+    pub fn descriptor(&self) -> &D {
+        &self.descriptor
+    }
 }
 
 impl<D> Iterator for SpkIterator<D>
@@ -148,7 +153,7 @@ mod test {
         Descriptor<DescriptorPublicKey>,
         Descriptor<DescriptorPublicKey>,
     ) {
-        let mut txout_index = KeychainTxOutIndex::<TestKeychain>::default();
+        let mut txout_index = KeychainTxOutIndex::<TestKeychain>::new(0);
 
         let secp = Secp256k1::signing_only();
         let (external_descriptor,_) = Descriptor::<DescriptorPublicKey>::parse_descriptor(&secp, "tr([73c5da0a/86'/0'/0']xprv9xgqHN7yz9MwCkxsBPN5qetuNdQSUttZNKw1dcYTV4mkaAFiBVGQziHs3NRSWMkCzvgjEe3n9xV8oYywvM8at9yRqyaZVz6TYYhX98VjsUk/0/*)").unwrap();

crates/chain/src/spk_txout_index.rs

@@ -168,9 +168,7 @@ impl<I: Clone + Ord> SpkTxOutIndex<I> {
     ///
     /// Returns `None` if the `TxOut` hasn't been scanned or if nothing matching was found there.
     pub fn txout(&self, outpoint: OutPoint) -> Option<(&I, &TxOut)> {
-        self.txouts
-            .get(&outpoint)
-            .map(|(spk_i, txout)| (spk_i, txout))
+        self.txouts.get(&outpoint).map(|v| (&v.0, &v.1))
     }
 
     /// Returns the script that has been inserted at the `index`.
@@ -217,7 +215,7 @@ impl<I: Clone + Ord> SpkTxOutIndex<I> {
     /// let unused_change_spks =
     ///     txout_index.unused_spks((change_index, u32::MIN)..(change_index, u32::MAX));
     /// ```
-    pub fn unused_spks<R>(&self, range: R) -> impl DoubleEndedIterator<Item = (&I, &Script)>
+    pub fn unused_spks<R>(&self, range: R) -> impl DoubleEndedIterator<Item = (&I, &Script)> + Clone
     where
         R: RangeBounds<I>,
     {

crates/chain/src/tx_data_traits.rs

@@ -123,8 +123,10 @@ pub trait Append {
 }
 
 impl<K: Ord, V> Append for BTreeMap<K, V> {
-    fn append(&mut self, mut other: Self) {
-        BTreeMap::append(self, &mut other)
+    fn append(&mut self, other: Self) {
+        // We use `extend` instead of `BTreeMap::append` due to performance issues with `append`.
+        // Refer to https://github.com/rust-lang/rust/issues/34666#issuecomment-675658420
+        BTreeMap::extend(self, other)
     }
 
     fn is_empty(&self) -> bool {
@@ -133,8 +135,10 @@ impl<K: Ord, V> Append for BTreeMap<K, V> {
 }
 
 impl<T: Ord> Append for BTreeSet<T> {
-    fn append(&mut self, mut other: Self) {
-        BTreeSet::append(self, &mut other)
+    fn append(&mut self, other: Self) {
+        // We use `extend` instead of `BTreeMap::append` due to performance issues with `append`.
+        // Refer to https://github.com/rust-lang/rust/issues/34666#issuecomment-675658420
+        BTreeSet::extend(self, other)
     }
 
     fn is_empty(&self) -> bool {

crates/chain/src/tx_graph.rs

@@ -581,10 +581,7 @@ impl<A: Clone + Ord> TxGraph<A> {
         }
 
         for (outpoint, txout) in changeset.txouts {
-            let tx_entry = self
-                .txs
-                .entry(outpoint.txid)
-                .or_insert_with(Default::default);
+            let tx_entry = self.txs.entry(outpoint.txid).or_default();
 
             match tx_entry {
                 (TxNodeInternal::Whole(_), _, _) => { /* do nothing since we already have full tx */
@@ -597,13 +594,13 @@ impl<A: Clone + Ord> TxGraph<A> {
 
         for (anchor, txid) in changeset.anchors {
             if self.anchors.insert((anchor.clone(), txid)) {
-                let (_, anchors, _) = self.txs.entry(txid).or_insert_with(Default::default);
+                let (_, anchors, _) = self.txs.entry(txid).or_default();
                 anchors.insert(anchor);
             }
         }
 
         for (txid, new_last_seen) in changeset.last_seen {
-            let (_, _, last_seen) = self.txs.entry(txid).or_insert_with(Default::default);
+            let (_, _, last_seen) = self.txs.entry(txid).or_default();
             if new_last_seen > *last_seen {
                 *last_seen = new_last_seen;
             }
@@ -1274,10 +1271,12 @@ impl<A> ChangeSet<A> {
 }
 
 impl<A: Ord> Append for ChangeSet<A> {
-    fn append(&mut self, mut other: Self) {
-        self.txs.append(&mut other.txs);
-        self.txouts.append(&mut other.txouts);
-        self.anchors.append(&mut other.anchors);
+    fn append(&mut self, other: Self) {
+        // We use `extend` instead of `BTreeMap::append` due to performance issues with `append`.
+        // Refer to https://github.com/rust-lang/rust/issues/34666#issuecomment-675658420
+        self.txs.extend(other.txs);
+        self.txouts.extend(other.txouts);
+        self.anchors.extend(other.anchors);
 
         // last_seen timestamps should only increase
         self.last_seen.extend(

crates/chain/tests/common/mod.rs

@@ -1,4 +1,5 @@
 mod tx_template;
+#[allow(unused_imports)]
 pub use tx_template::*;
 
 #[allow(unused_macros)]

crates/chain/tests/test_indexed_tx_graph.rs

@@ -27,9 +27,10 @@ fn insert_relevant_txs() {
     let spk_0 = descriptor.at_derivation_index(0).unwrap().script_pubkey();
     let spk_1 = descriptor.at_derivation_index(9).unwrap().script_pubkey();
 
-    let mut graph = IndexedTxGraph::<ConfirmationHeightAnchor, KeychainTxOutIndex<()>>::default();
+    let mut graph = IndexedTxGraph::<ConfirmationHeightAnchor, KeychainTxOutIndex<()>>::new(
+        KeychainTxOutIndex::new(10),
+    );
     graph.index.add_keychain((), descriptor);
-    graph.index.set_lookahead(&(), 10);
 
     let tx_a = Transaction {
         output: vec![
@@ -118,12 +119,12 @@ fn test_list_owned_txouts() {
     let (desc_1, _) = Descriptor::parse_descriptor(&Secp256k1::signing_only(), "tr(tprv8ZgxMBicQKsPd3krDUsBAmtnRsK3rb8u5yi1zhQgMhF1tR8MW7xfE4rnrbbsrbPR52e7rKapu6ztw1jXveJSCGHEriUGZV7mCe88duLp5pj/86'/1'/0'/0/*)").unwrap();
     let (desc_2, _) = Descriptor::parse_descriptor(&Secp256k1::signing_only(), "tr(tprv8ZgxMBicQKsPd3krDUsBAmtnRsK3rb8u5yi1zhQgMhF1tR8MW7xfE4rnrbbsrbPR52e7rKapu6ztw1jXveJSCGHEriUGZV7mCe88duLp5pj/86'/1'/0'/1/*)").unwrap();
 
-    let mut graph =
-        IndexedTxGraph::<ConfirmationHeightAnchor, KeychainTxOutIndex<String>>::default();
+    let mut graph = IndexedTxGraph::<ConfirmationHeightAnchor, KeychainTxOutIndex<String>>::new(
+        KeychainTxOutIndex::new(10),
+    );
 
     graph.index.add_keychain("keychain_1".into(), desc_1);
     graph.index.add_keychain("keychain_2".into(), desc_2);
-    graph.index.set_lookahead_for_all(10);
 
     // Get trusted and untrusted addresses
 

crates/chain/tests/test_keychain_txout_index.rs

@@ -18,12 +18,14 @@ enum TestKeychain {
     Internal,
 }
 
-fn init_txout_index() -> (
+fn init_txout_index(
+    lookahead: u32,
+) -> (
     bdk_chain::keychain::KeychainTxOutIndex<TestKeychain>,
     Descriptor<DescriptorPublicKey>,
     Descriptor<DescriptorPublicKey>,
 ) {
-    let mut txout_index = bdk_chain::keychain::KeychainTxOutIndex::<TestKeychain>::default();
+    let mut txout_index = bdk_chain::keychain::KeychainTxOutIndex::<TestKeychain>::new(lookahead);
 
     let secp = bdk_chain::bitcoin::secp256k1::Secp256k1::signing_only();
     let (external_descriptor,_) = Descriptor::<DescriptorPublicKey>::parse_descriptor(&secp, "tr([73c5da0a/86'/0'/0']xprv9xgqHN7yz9MwCkxsBPN5qetuNdQSUttZNKw1dcYTV4mkaAFiBVGQziHs3NRSWMkCzvgjEe3n9xV8oYywvM8at9yRqyaZVz6TYYhX98VjsUk/0/*)").unwrap();
@@ -46,7 +48,7 @@ fn spk_at_index(descriptor: &Descriptor<DescriptorPublicKey>, index: u32) -> Scr
 fn test_set_all_derivation_indices() {
     use bdk_chain::indexed_tx_graph::Indexer;
 
-    let (mut txout_index, _, _) = init_txout_index();
+    let (mut txout_index, _, _) = init_txout_index(0);
     let derive_to: BTreeMap<_, _> =
         [(TestKeychain::External, 12), (TestKeychain::Internal, 24)].into();
     assert_eq!(
@@ -64,19 +66,10 @@ fn test_set_all_derivation_indices() {
 
 #[test]
 fn test_lookahead() {
-    let (mut txout_index, external_desc, internal_desc) = init_txout_index();
-
-    // ensure it does not break anything if lookahead is set multiple times
-    (0..=10).for_each(|lookahead| txout_index.set_lookahead(&TestKeychain::External, lookahead));
-    (0..=20)
-        .filter(|v| v % 2 == 0)
-        .for_each(|lookahead| txout_index.set_lookahead(&TestKeychain::Internal, lookahead));
-
-    assert_eq!(txout_index.inner().all_spks().len(), 30);
+    let (mut txout_index, external_desc, internal_desc) = init_txout_index(10);
 
     // given:
     // - external lookahead set to 10
-    // - internal lookahead set to 20
     // when:
     // - set external derivation index to value higher than last, but within the lookahead value
     // expect:
@@ -97,37 +90,37 @@ fn test_lookahead() {
         assert_eq!(
             txout_index.inner().all_spks().len(),
             10 /* external lookahead */ +
-            20 /* internal lookahead */ +
+            10 /* internal lookahead */ +
             index as usize + 1 /* `derived` count */
         );
         assert_eq!(
             txout_index
-                .revealed_spks_of_keychain(&TestKeychain::External)
+                .revealed_keychain_spks(&TestKeychain::External)
                 .count(),
             index as usize + 1,
         );
         assert_eq!(
             txout_index
-                .revealed_spks_of_keychain(&TestKeychain::Internal)
+                .revealed_keychain_spks(&TestKeychain::Internal)
                 .count(),
             0,
         );
         assert_eq!(
             txout_index
-                .unused_spks_of_keychain(&TestKeychain::External)
+                .unused_keychain_spks(&TestKeychain::External)
                 .count(),
             index as usize + 1,
         );
         assert_eq!(
             txout_index
-                .unused_spks_of_keychain(&TestKeychain::Internal)
+                .unused_keychain_spks(&TestKeychain::Internal)
                 .count(),
             0,
         );
     }
 
     // given:
-    // - internal lookahead is 20
+    // - internal lookahead is 10
     // - internal derivation index is `None`
     // when:
     // - derivation index is set ahead of current derivation index + lookahead
@@ -148,13 +141,13 @@ fn test_lookahead() {
     assert_eq!(
         txout_index.inner().all_spks().len(),
         10 /* external lookahead */ +
-        20 /* internal lookahead */ +
+        10 /* internal lookahead */ +
         20 /* external stored index count */ +
         25 /* internal stored index count */
     );
     assert_eq!(
         txout_index
-            .revealed_spks_of_keychain(&TestKeychain::Internal)
+            .revealed_keychain_spks(&TestKeychain::Internal)
             .count(),
         25,
     );
@@ -206,13 +199,13 @@ fn test_lookahead() {
         );
         assert_eq!(
             txout_index
-                .revealed_spks_of_keychain(&TestKeychain::External)
+                .revealed_keychain_spks(&TestKeychain::External)
                 .count(),
             last_external_index as usize + 1,
         );
         assert_eq!(
             txout_index
-                .revealed_spks_of_keychain(&TestKeychain::Internal)
+                .revealed_keychain_spks(&TestKeychain::Internal)
                 .count(),
             last_internal_index as usize + 1,
         );
@@ -226,8 +219,7 @@ fn test_lookahead() {
 // - last used index should change as expected
 #[test]
 fn test_scan_with_lookahead() {
-    let (mut txout_index, external_desc, _) = init_txout_index();
-    txout_index.set_lookahead_for_all(10);
+    let (mut txout_index, external_desc, _) = init_txout_index(10);
 
     let spks: BTreeMap<u32, ScriptBuf> = [0, 10, 20, 30]
         .into_iter()
@@ -281,7 +273,7 @@ fn test_scan_with_lookahead() {
 #[test]
 #[rustfmt::skip]
 fn test_wildcard_derivations() {
-    let (mut txout_index, external_desc, _) = init_txout_index();
+    let (mut txout_index, external_desc, _) = init_txout_index(0);
     let external_spk_0 = external_desc.at_derivation_index(0).unwrap().script_pubkey();
     let external_spk_16 = external_desc.at_derivation_index(16).unwrap().script_pubkey();
     let external_spk_26 = external_desc.at_derivation_index(26).unwrap().script_pubkey();
@@ -313,7 +305,7 @@ fn test_wildcard_derivations() {
 
     (0..=15)
         .chain([17, 20, 23])
-        .for_each(|index| assert!(txout_index.mark_used(&TestKeychain::External, index)));
+        .for_each(|index| assert!(txout_index.mark_used(TestKeychain::External, index)));
 
     assert_eq!(txout_index.next_index(&TestKeychain::External), (26, true));
 
@@ -329,7 +321,7 @@ fn test_wildcard_derivations() {
     // - Use all the derived till 26.
     // - next_unused() = ((27, <spk>), keychain::ChangeSet)
     (0..=26).for_each(|index| {
-        txout_index.mark_used(&TestKeychain::External, index);
+        txout_index.mark_used(TestKeychain::External, index);
     });
 
     let (spk, changeset) = txout_index.next_unused_spk(&TestKeychain::External);
@@ -339,7 +331,7 @@ fn test_wildcard_derivations() {
 
 #[test]
 fn test_non_wildcard_derivations() {
-    let mut txout_index = KeychainTxOutIndex::<TestKeychain>::default();
+    let mut txout_index = KeychainTxOutIndex::<TestKeychain>::new(0);
 
     let secp = bitcoin::secp256k1::Secp256k1::signing_only();
     let (no_wildcard_descriptor, _) = Descriptor::<DescriptorPublicKey>::parse_descriptor(&secp, "wpkh([73c5da0a/86'/0'/0']xprv9xgqHN7yz9MwCkxsBPN5qetuNdQSUttZNKw1dcYTV4mkaAFiBVGQziHs3NRSWMkCzvgjEe3n9xV8oYywvM8at9yRqyaZVz6TYYhX98VjsUk/1/0)").unwrap();
@@ -372,7 +364,7 @@ fn test_non_wildcard_derivations() {
     // - derive new and next unused should return the old script
     // - store_up_to should not panic and return empty changeset
     assert_eq!(txout_index.next_index(&TestKeychain::External), (0, false));
-    txout_index.mark_used(&TestKeychain::External, 0);
+    txout_index.mark_used(TestKeychain::External, 0);
 
     let (spk, changeset) = txout_index.reveal_next_spk(&TestKeychain::External);
     assert_eq!(spk, (0, external_spk.as_script()));
@@ -389,7 +381,7 @@ fn test_non_wildcard_derivations() {
     // we check that spks_of_keychain returns a SpkIterator with just one element
     assert_eq!(
         txout_index
-            .spks_of_keychain(&TestKeychain::External)
+            .revealed_keychain_spks(&TestKeychain::External)
             .count(),
         1,
     );

crates/chain/tests/test_local_chain.rs

@@ -1,5 +1,5 @@
 use bdk_chain::local_chain::{
-    AlterCheckPointError, CannotConnectError, ChangeSet, LocalChain, Update,
+    AlterCheckPointError, CannotConnectError, ChangeSet, LocalChain, MissingGenesisError, Update,
 };
 use bitcoin::BlockHash;
 
@@ -350,3 +350,76 @@ fn local_chain_insert_block() {
         assert_eq!(chain, t.expected_final, "[{}] unexpected final chain", i,);
     }
 }
+
+#[test]
+fn local_chain_disconnect_from() {
+    struct TestCase {
+        name: &'static str,
+        original: LocalChain,
+        disconnect_from: (u32, BlockHash),
+        exp_result: Result<ChangeSet, MissingGenesisError>,
+        exp_final: LocalChain,
+    }
+
+    let test_cases = [
+        TestCase {
+            name: "try_replace_genesis_should_fail",
+            original: local_chain![(0, h!("_"))],
+            disconnect_from: (0, h!("_")),
+            exp_result: Err(MissingGenesisError),
+            exp_final: local_chain![(0, h!("_"))],
+        },
+        TestCase {
+            name: "try_replace_genesis_should_fail_2",
+            original: local_chain![(0, h!("_")), (2, h!("B")), (3, h!("C"))],
+            disconnect_from: (0, h!("_")),
+            exp_result: Err(MissingGenesisError),
+            exp_final: local_chain![(0, h!("_")), (2, h!("B")), (3, h!("C"))],
+        },
+        TestCase {
+            name: "from_does_not_exist",
+            original: local_chain![(0, h!("_")), (3, h!("C"))],
+            disconnect_from: (2, h!("B")),
+            exp_result: Ok(ChangeSet::default()),
+            exp_final: local_chain![(0, h!("_")), (3, h!("C"))],
+        },
+        TestCase {
+            name: "from_has_different_blockhash",
+            original: local_chain![(0, h!("_")), (2, h!("B"))],
+            disconnect_from: (2, h!("not_B")),
+            exp_result: Ok(ChangeSet::default()),
+            exp_final: local_chain![(0, h!("_")), (2, h!("B"))],
+        },
+        TestCase {
+            name: "disconnect_one",
+            original: local_chain![(0, h!("_")), (2, h!("B"))],
+            disconnect_from: (2, h!("B")),
+            exp_result: Ok(ChangeSet::from_iter([(2, None)])),
+            exp_final: local_chain![(0, h!("_"))],
+        },
+        TestCase {
+            name: "disconnect_three",
+            original: local_chain![(0, h!("_")), (2, h!("B")), (3, h!("C")), (4, h!("D"))],
+            disconnect_from: (2, h!("B")),
+            exp_result: Ok(ChangeSet::from_iter([(2, None), (3, None), (4, None)])),
+            exp_final: local_chain![(0, h!("_"))],
+        },
+    ];
+
+    for (i, t) in test_cases.into_iter().enumerate() {
+        println!("Case {}: {}", i, t.name);
+
+        let mut chain = t.original;
+        let result = chain.disconnect_from(t.disconnect_from.into());
+        assert_eq!(
+            result, t.exp_result,
+            "[{}:{}] unexpected changeset result",
+            i, t.name
+        );
+        assert_eq!(
+            chain, t.exp_final,
+            "[{}:{}] unexpected final chain",
+            i, t.name
+        );
+    }
+}

crates/electrum/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "bdk_electrum"
-version = "0.4.0"
+version = "0.6.0"
 edition = "2021"
 homepage = "https://bitcoindevkit.org"
 repository = "https://github.com/bitcoindevkit/bdk"
@@ -12,6 +12,6 @@ readme = "README.md"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-bdk_chain = { path = "../chain", version = "0.6.0", default-features = false }
+bdk_chain = { path = "../chain", version = "0.8.0", default-features = false }
 electrum-client = { version = "0.18" }
 #rustls = { version = "=0.21.1", optional = true, features = ["dangerous_configuration"] }

crates/electrum/README.md

@@ -1,3 +1,7 @@
 # BDK Electrum
 
-BDK Electrum client library for updating the keychain tracker.
+BDK Electrum extends [`electrum-client`] to update [`bdk_chain`] structures
+from an Electrum server.
+
+[`electrum-client`]: https://docs.rs/electrum-client/
+[`bdk_chain`]: https://docs.rs/bdk-chain/

crates/electrum/src/electrum_ext.rs

@@ -134,76 +134,63 @@ pub struct ElectrumUpdate {
 
 /// Trait to extend [`Client`] functionality.
 pub trait ElectrumExt {
-    /// Scan the blockchain (via electrum) for the data specified and returns updates for
-    /// [`bdk_chain`] data structures.
+    /// Full scan the keychain scripts specified with the blockchain (via an Electrum client) and
+    /// returns updates for [`bdk_chain`] data structures.
     ///
     /// - `prev_tip`: the most recent blockchain tip present locally
     /// - `keychain_spks`: keychains that we want to scan transactions for
-    /// - `txids`: transactions for which we want updated [`Anchor`]s
-    /// - `outpoints`: transactions associated with these outpoints (residing, spending) that we
-    ///     want to included in the update
     ///
-    /// The scan for each keychain stops after a gap of `stop_gap` script pubkeys with no associated
+    /// The full scan for each keychain stops after a gap of `stop_gap` script pubkeys with no associated
     /// transactions. `batch_size` specifies the max number of script pubkeys to request for in a
     /// single batch request.
-    fn scan<K: Ord + Clone>(
+    fn full_scan<K: Ord + Clone>(
         &self,
         prev_tip: CheckPoint,
         keychain_spks: BTreeMap<K, impl IntoIterator<Item = (u32, ScriptBuf)>>,
-        txids: impl IntoIterator<Item = Txid>,
-        outpoints: impl IntoIterator<Item = OutPoint>,
         stop_gap: usize,
         batch_size: usize,
     ) -> Result<(ElectrumUpdate, BTreeMap<K, u32>), Error>;
 
-    /// Convenience method to call [`scan`] without requiring a keychain.
+    /// Sync a set of scripts with the blockchain (via an Electrum client) for the data specified
+    /// and returns updates for [`bdk_chain`] data structures.
     ///
-    /// [`scan`]: ElectrumExt::scan
-    fn scan_without_keychain(
+    /// - `prev_tip`: the most recent blockchain tip present locally
+    /// - `misc_spks`: an iterator of scripts we want to sync transactions for
+    /// - `txids`: transactions for which we want updated [`Anchor`]s
+    /// - `outpoints`: transactions associated with these outpoints (residing, spending) that we
+    ///     want to include in the update
+    ///
+    /// `batch_size` specifies the max number of script pubkeys to request for in a single batch
+    /// request.
+    ///
+    /// If the scripts to sync are unknown, such as when restoring or importing a keychain that
+    /// may include scripts that have been used, use [`full_scan`] with the keychain.
+    ///
+    /// [`full_scan`]: ElectrumExt::full_scan
+    fn sync(
         &self,
         prev_tip: CheckPoint,
         misc_spks: impl IntoIterator<Item = ScriptBuf>,
         txids: impl IntoIterator<Item = Txid>,
         outpoints: impl IntoIterator<Item = OutPoint>,
         batch_size: usize,
-    ) -> Result<ElectrumUpdate, Error> {
-        let spk_iter = misc_spks
-            .into_iter()
-            .enumerate()
-            .map(|(i, spk)| (i as u32, spk));
-
-        let (electrum_update, _) = self.scan(
-            prev_tip,
-            [((), spk_iter)].into(),
-            txids,
-            outpoints,
-            usize::MAX,
-            batch_size,
-        )?;
-
-        Ok(electrum_update)
-    }
+    ) -> Result<ElectrumUpdate, Error>;
 }
 
 impl ElectrumExt for Client {
-    fn scan<K: Ord + Clone>(
+    fn full_scan<K: Ord + Clone>(
         &self,
         prev_tip: CheckPoint,
         keychain_spks: BTreeMap<K, impl IntoIterator<Item = (u32, ScriptBuf)>>,
-        txids: impl IntoIterator<Item = Txid>,
-        outpoints: impl IntoIterator<Item = OutPoint>,
         stop_gap: usize,
         batch_size: usize,
     ) -> Result<(ElectrumUpdate, BTreeMap<K, u32>), Error> {
         let mut request_spks = keychain_spks
             .into_iter()
-            .map(|(k, s)| (k, s.into_iter()))
+            .map(|(k, s)| (k.clone(), s.into_iter()))
             .collect::<BTreeMap<K, _>>();
         let mut scanned_spks = BTreeMap::<(K, u32), (ScriptBuf, bool)>::new();
 
-        let txids = txids.into_iter().collect::<Vec<_>>();
-        let outpoints = outpoints.into_iter().collect::<Vec<_>>();
-
         let (electrum_update, keychain_update) = loop {
             let (tip, _) = construct_update_tip(self, prev_tip.clone())?;
             let mut relevant_txids = RelevantTxids::default();
@@ -242,15 +229,6 @@ impl ElectrumExt for Client {
                 }
             }
 
-            populate_with_txids(self, &cps, &mut relevant_txids, &mut txids.iter().cloned())?;
-
-            let _txs = populate_with_outpoints(
-                self,
-                &cps,
-                &mut relevant_txids,
-                &mut outpoints.iter().cloned(),
-            )?;
-
             // check for reorgs during scan process
             let server_blockhash = self.block_header(tip.height() as usize)?.block_hash();
             if tip.hash() != server_blockhash {
@@ -284,6 +262,41 @@ impl ElectrumExt for Client {
 
         Ok((electrum_update, keychain_update))
     }
+
+    fn sync(
+        &self,
+        prev_tip: CheckPoint,
+        misc_spks: impl IntoIterator<Item = ScriptBuf>,
+        txids: impl IntoIterator<Item = Txid>,
+        outpoints: impl IntoIterator<Item = OutPoint>,
+        batch_size: usize,
+    ) -> Result<ElectrumUpdate, Error> {
+        let spk_iter = misc_spks
+            .into_iter()
+            .enumerate()
+            .map(|(i, spk)| (i as u32, spk));
+
+        let (mut electrum_update, _) = self.full_scan(
+            prev_tip.clone(),
+            [((), spk_iter)].into(),
+            usize::MAX,
+            batch_size,
+        )?;
+
+        let (tip, _) = construct_update_tip(self, prev_tip)?;
+        let cps = tip
+            .iter()
+            .take(10)
+            .map(|cp| (cp.height(), cp))
+            .collect::<BTreeMap<u32, CheckPoint>>();
+
+        populate_with_txids(self, &cps, &mut electrum_update.relevant_txids, txids)?;
+
+        let _txs =
+            populate_with_outpoints(self, &cps, &mut electrum_update.relevant_txids, outpoints)?;
+
+        Ok(electrum_update)
+    }
 }
 
 /// Return a [`CheckPoint`] of the latest tip, that connects with `prev_tip`.
@@ -405,7 +418,7 @@ fn populate_with_outpoints(
     client: &Client,
     cps: &BTreeMap<u32, CheckPoint>,
     relevant_txids: &mut RelevantTxids,
-    outpoints: &mut impl Iterator<Item = OutPoint>,
+    outpoints: impl IntoIterator<Item = OutPoint>,
 ) -> Result<HashMap<Txid, Transaction>, Error> {
     let mut full_txs = HashMap::new();
     for outpoint in outpoints {
@@ -466,7 +479,7 @@ fn populate_with_txids(
     client: &Client,
     cps: &BTreeMap<u32, CheckPoint>,
     relevant_txids: &mut RelevantTxids,
-    txids: &mut impl Iterator<Item = Txid>,
+    txids: impl IntoIterator<Item = Txid>,
 ) -> Result<(), Error> {
     for txid in txids {
         let tx = match client.transaction_get(&txid) {
@@ -477,7 +490,7 @@ fn populate_with_txids(
 
         let spk = tx
             .output
-            .get(0)
+            .first()
             .map(|txo| &txo.script_pubkey)
             .expect("tx must have an output");
 

crates/electrum/src/lib.rs

@@ -1,26 +1,26 @@
-//! This crate is used for updating structures of the [`bdk_chain`] crate with data from electrum.
+//! This crate is used for updating structures of [`bdk_chain`] with data from an Electrum server.
 //!
-//! The star of the show is the [`ElectrumExt::scan`] method, which scans for relevant blockchain
-//! data (via electrum) and outputs updates for [`bdk_chain`] structures as a tuple of form:
+//! The two primary methods are [`ElectrumExt::sync`] and [`ElectrumExt::full_scan`]. In most cases
+//! [`ElectrumExt::sync`] is used to sync the transaction histories of scripts that the application
+//! cares about, for example the scripts for all the receive addresses of a Wallet's keychain that it
+//! has shown a user. [`ElectrumExt::full_scan`] is meant to be used when importing or restoring a
+//! keychain where the range of possibly used scripts is not known. In this case it is necessary to
+//! scan all keychain scripts until a number (the "stop gap") of unused scripts is discovered. For a
+//! sync or full scan the user receives relevant blockchain data and output updates for
+//! [`bdk_chain`] including [`RelevantTxids`].
 //!
-//! ([`bdk_chain::local_chain::Update`], [`RelevantTxids`], `keychain_update`)
+//! The [`RelevantTxids`] only includes `txid`s and not full transactions. The caller is responsible
+//! for obtaining full transactions before applying new data to their [`bdk_chain`]. This can be
+//! done with these steps:
 //!
-//! An [`RelevantTxids`] only includes `txid`s and no full transactions. The caller is
-//! responsible for obtaining full transactions before applying. This can be done with
-//! these steps:
+//! 1. Determine which full transactions are missing. Use [`RelevantTxids::missing_full_txs`].
 //!
-//! 1. Determine which full transactions are missing. The method [`missing_full_txs`] of
-//! [`RelevantTxids`] can be used.
+//! 2. Obtaining the full transactions. To do this via electrum use [`ElectrumApi::batch_transaction_get`].
 //!
-//! 2. Obtaining the full transactions. To do this via electrum, the method
-//! [`batch_transaction_get`] can be used.
+//! Refer to [`example_electrum`] for a complete example.
 //!
-//! Refer to [`bdk_electrum_example`] for a complete example.
-//!
-//! [`ElectrumClient::scan`]: electrum_client::ElectrumClient::scan
-//! [`missing_full_txs`]: RelevantTxids::missing_full_txs
-//! [`batch_transaction_get`]: electrum_client::ElectrumApi::batch_transaction_get
-//! [`bdk_electrum_example`]: https://github.com/LLFourn/bdk_core_staging/tree/master/bdk_electrum_example
+//! [`ElectrumApi::batch_transaction_get`]: electrum_client::ElectrumApi::batch_transaction_get
+//! [`example_electrum`]: https://github.com/bitcoindevkit/bdk/tree/master/example-crates/example_electrum
 
 #![warn(missing_docs)]
 

crates/esplora/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "bdk_esplora"
-version = "0.4.0"
+version = "0.6.0"
 edition = "2021"
 homepage = "https://bitcoindevkit.org"
 repository = "https://github.com/bitcoindevkit/bdk"
@@ -12,7 +12,7 @@ readme = "README.md"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-bdk_chain = { path = "../chain", version = "0.6.0", default-features = false }
+bdk_chain = { path = "../chain", version = "0.8.0", default-features = false }
 esplora-client = { version = "0.6.0", default-features = false }
 async-trait = { version = "0.1.66", optional = true }
 futures = { version = "0.3.26", optional = true }

crates/esplora/src/async_ext.rs

@@ -36,58 +36,45 @@ pub trait EsploraAsyncExt {
         request_heights: impl IntoIterator<IntoIter = impl Iterator<Item = u32> + Send> + Send,
     ) -> Result<local_chain::Update, Error>;
 
-    /// Scan Esplora for the data specified and return a [`TxGraph`] and a map of last active
-    /// indices.
+    /// Full scan the keychain scripts specified with the blockchain (via an Esplora client) and
+    /// returns a [`TxGraph`] and a map of last active indices.
     ///
     /// * `keychain_spks`: keychains that we want to scan transactions for
-    /// * `txids`: transactions for which we want updated [`ConfirmationTimeHeightAnchor`]s
-    /// * `outpoints`: transactions associated with these outpoints (residing, spending) that we
-    ///     want to include in the update
     ///
-    /// The scan for each keychain stops after a gap of `stop_gap` script pubkeys with no associated
+    /// The full scan for each keychain stops after a gap of `stop_gap` script pubkeys with no associated
     /// transactions. `parallel_requests` specifies the max number of HTTP requests to make in
     /// parallel.
     #[allow(clippy::result_large_err)]
-    async fn scan_txs_with_keychains<K: Ord + Clone + Send>(
+    async fn full_scan<K: Ord + Clone + Send>(
         &self,
         keychain_spks: BTreeMap<
             K,
             impl IntoIterator<IntoIter = impl Iterator<Item = (u32, ScriptBuf)> + Send> + Send,
         >,
-        txids: impl IntoIterator<IntoIter = impl Iterator<Item = Txid> + Send> + Send,
-        outpoints: impl IntoIterator<IntoIter = impl Iterator<Item = OutPoint> + Send> + Send,
         stop_gap: usize,
         parallel_requests: usize,
     ) -> Result<(TxGraph<ConfirmationTimeHeightAnchor>, BTreeMap<K, u32>), Error>;
 
-    /// Convenience method to call [`scan_txs_with_keychains`] without requiring a keychain.
+    /// Sync a set of scripts with the blockchain (via an Esplora client) for the data
+    /// specified and return a [`TxGraph`].
     ///
-    /// [`scan_txs_with_keychains`]: EsploraAsyncExt::scan_txs_with_keychains
+    /// * `misc_spks`: scripts that we want to sync transactions for
+    /// * `txids`: transactions for which we want updated [`ConfirmationTimeHeightAnchor`]s
+    /// * `outpoints`: transactions associated with these outpoints (residing, spending) that we
+    ///     want to include in the update
+    ///
+    /// If the scripts to sync are unknown, such as when restoring or importing a keychain that
+    /// may include scripts that have been used, use [`full_scan`] with the keychain.
+    ///
+    /// [`full_scan`]: EsploraAsyncExt::full_scan
     #[allow(clippy::result_large_err)]
-    async fn scan_txs(
+    async fn sync(
         &self,
         misc_spks: impl IntoIterator<IntoIter = impl Iterator<Item = ScriptBuf> + Send> + Send,
         txids: impl IntoIterator<IntoIter = impl Iterator<Item = Txid> + Send> + Send,
         outpoints: impl IntoIterator<IntoIter = impl Iterator<Item = OutPoint> + Send> + Send,
         parallel_requests: usize,
-    ) -> Result<TxGraph<ConfirmationTimeHeightAnchor>, Error> {
-        self.scan_txs_with_keychains(
-            [(
-                (),
-                misc_spks
-                    .into_iter()
-                    .enumerate()
-                    .map(|(i, spk)| (i as u32, spk)),
-            )]
-            .into(),
-            txids,
-            outpoints,
-            usize::MAX,
-            parallel_requests,
-        )
-        .await
-        .map(|(g, _)| g)
-    }
+    ) -> Result<TxGraph<ConfirmationTimeHeightAnchor>, Error>;
 }
 
 #[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
@@ -199,14 +186,12 @@ impl EsploraAsyncExt for esplora_client::AsyncClient {
         })
     }
 
-    async fn scan_txs_with_keychains<K: Ord + Clone + Send>(
+    async fn full_scan<K: Ord + Clone + Send>(
         &self,
         keychain_spks: BTreeMap<
             K,
             impl IntoIterator<IntoIter = impl Iterator<Item = (u32, ScriptBuf)> + Send> + Send,
         >,
-        txids: impl IntoIterator<IntoIter = impl Iterator<Item = Txid> + Send> + Send,
-        outpoints: impl IntoIterator<IntoIter = impl Iterator<Item = OutPoint> + Send> + Send,
         stop_gap: usize,
         parallel_requests: usize,
     ) -> Result<(TxGraph<ConfirmationTimeHeightAnchor>, BTreeMap<K, u32>), Error> {
@@ -275,6 +260,32 @@ impl EsploraAsyncExt for esplora_client::AsyncClient {
             }
         }
 
+        Ok((graph, last_active_indexes))
+    }
+
+    async fn sync(
+        &self,
+        misc_spks: impl IntoIterator<IntoIter = impl Iterator<Item = ScriptBuf> + Send> + Send,
+        txids: impl IntoIterator<IntoIter = impl Iterator<Item = Txid> + Send> + Send,
+        outpoints: impl IntoIterator<IntoIter = impl Iterator<Item = OutPoint> + Send> + Send,
+        parallel_requests: usize,
+    ) -> Result<TxGraph<ConfirmationTimeHeightAnchor>, Error> {
+        let mut graph = self
+            .full_scan(
+                [(
+                    (),
+                    misc_spks
+                        .into_iter()
+                        .enumerate()
+                        .map(|(i, spk)| (i as u32, spk)),
+                )]
+                .into(),
+                usize::MAX,
+                parallel_requests,
+            )
+            .await
+            .map(|(g, _)| g)?;
+
         let mut txids = txids.into_iter();
         loop {
             let handles = txids
@@ -323,7 +334,6 @@ impl EsploraAsyncExt for esplora_client::AsyncClient {
                 }
             }
         }
-
-        Ok((graph, last_active_indexes))
+        Ok(graph)
     }
 }

crates/esplora/src/blocking_ext.rs

@@ -19,8 +19,8 @@ use crate::{anchor_from_status, ASSUME_FINAL_DEPTH};
 pub trait EsploraExt {
     /// Prepare an [`LocalChain`] update with blocks fetched from Esplora.
     ///
-    /// * `prev_tip` is the previous tip of [`LocalChain::tip`].
-    /// * `get_heights` is the block heights that we are interested in fetching from Esplora.
+    /// * `local_tip` is the previous tip of [`LocalChain::tip`].
+    /// * `request_heights` is the block heights that we are interested in fetching from Esplora.
     ///
     /// The result of this method can be applied to [`LocalChain::apply_update`].
     ///
@@ -34,54 +34,42 @@ pub trait EsploraExt {
         request_heights: impl IntoIterator<Item = u32>,
     ) -> Result<local_chain::Update, Error>;
 
-    /// Scan Esplora for the data specified and return a [`TxGraph`] and a map of last active
-    /// indices.
+    /// Full scan the keychain scripts specified with the blockchain (via an Esplora client) and
+    /// returns a [`TxGraph`] and a map of last active indices.
     ///
     /// * `keychain_spks`: keychains that we want to scan transactions for
-    /// * `txids`: transactions for which we want updated [`ConfirmationTimeHeightAnchor`]s
-    /// * `outpoints`: transactions associated with these outpoints (residing, spending) that we
-    ///     want to include in the update
     ///
-    /// The scan for each keychain stops after a gap of `stop_gap` script pubkeys with no associated
+    /// The full scan for each keychain stops after a gap of `stop_gap` script pubkeys with no associated
     /// transactions. `parallel_requests` specifies the max number of HTTP requests to make in
     /// parallel.
     #[allow(clippy::result_large_err)]
-    fn scan_txs_with_keychains<K: Ord + Clone>(
+    fn full_scan<K: Ord + Clone>(
         &self,
         keychain_spks: BTreeMap<K, impl IntoIterator<Item = (u32, ScriptBuf)>>,
-        txids: impl IntoIterator<Item = Txid>,
-        outpoints: impl IntoIterator<Item = OutPoint>,
         stop_gap: usize,
         parallel_requests: usize,
     ) -> Result<(TxGraph<ConfirmationTimeHeightAnchor>, BTreeMap<K, u32>), Error>;
 
-    /// Convenience method to call [`scan_txs_with_keychains`] without requiring a keychain.
+    /// Sync a set of scripts with the blockchain (via an Esplora client) for the data
+    /// specified and return a [`TxGraph`].
     ///
-    /// [`scan_txs_with_keychains`]: EsploraExt::scan_txs_with_keychains
+    /// * `misc_spks`: scripts that we want to sync transactions for
+    /// * `txids`: transactions for which we want updated [`ConfirmationTimeHeightAnchor`]s
+    /// * `outpoints`: transactions associated with these outpoints (residing, spending) that we
+    ///     want to include in the update
+    ///
+    /// If the scripts to sync are unknown, such as when restoring or importing a keychain that
+    /// may include scripts that have been used, use [`full_scan`] with the keychain.
+    ///
+    /// [`full_scan`]: EsploraExt::full_scan
     #[allow(clippy::result_large_err)]
-    fn scan_txs(
+    fn sync(
         &self,
         misc_spks: impl IntoIterator<Item = ScriptBuf>,
         txids: impl IntoIterator<Item = Txid>,
         outpoints: impl IntoIterator<Item = OutPoint>,
         parallel_requests: usize,
-    ) -> Result<TxGraph<ConfirmationTimeHeightAnchor>, Error> {
-        self.scan_txs_with_keychains(
-            [(
-                (),
-                misc_spks
-                    .into_iter()
-                    .enumerate()
-                    .map(|(i, spk)| (i as u32, spk)),
-            )]
-            .into(),
-            txids,
-            outpoints,
-            usize::MAX,
-            parallel_requests,
-        )
-        .map(|(g, _)| g)
-    }
+    ) -> Result<TxGraph<ConfirmationTimeHeightAnchor>, Error>;
 }
 
 impl EsploraExt for esplora_client::BlockingClient {
@@ -190,11 +178,9 @@ impl EsploraExt for esplora_client::BlockingClient {
         })
     }
 
-    fn scan_txs_with_keychains<K: Ord + Clone>(
+    fn full_scan<K: Ord + Clone>(
         &self,
         keychain_spks: BTreeMap<K, impl IntoIterator<Item = (u32, ScriptBuf)>>,
-        txids: impl IntoIterator<Item = Txid>,
-        outpoints: impl IntoIterator<Item = OutPoint>,
         stop_gap: usize,
         parallel_requests: usize,
     ) -> Result<(TxGraph<ConfirmationTimeHeightAnchor>, BTreeMap<K, u32>), Error> {
@@ -266,6 +252,31 @@ impl EsploraExt for esplora_client::BlockingClient {
             }
         }
 
+        Ok((graph, last_active_indexes))
+    }
+
+    fn sync(
+        &self,
+        misc_spks: impl IntoIterator<Item = ScriptBuf>,
+        txids: impl IntoIterator<Item = Txid>,
+        outpoints: impl IntoIterator<Item = OutPoint>,
+        parallel_requests: usize,
+    ) -> Result<TxGraph<ConfirmationTimeHeightAnchor>, Error> {
+        let mut graph = self
+            .full_scan(
+                [(
+                    (),
+                    misc_spks
+                        .into_iter()
+                        .enumerate()
+                        .map(|(i, spk)| (i as u32, spk)),
+                )]
+                .into(),
+                usize::MAX,
+                parallel_requests,
+            )
+            .map(|(g, _)| g)?;
+
         let mut txids = txids.into_iter();
         loop {
             let handles = txids
@@ -292,7 +303,7 @@ impl EsploraExt for esplora_client::BlockingClient {
             }
         }
 
-        for op in outpoints.into_iter() {
+        for op in outpoints {
             if graph.get_tx(op.txid).is_none() {
                 if let Some(tx) = self.get_tx(&op.txid)? {
                     let _ = graph.insert_tx(tx);
@@ -317,7 +328,6 @@ impl EsploraExt for esplora_client::BlockingClient {
                 }
             }
         }
-
-        Ok((graph, last_active_indexes))
+        Ok(graph)
     }
 }

crates/esplora/src/lib.rs

@@ -1,4 +1,21 @@
 #![doc = include_str!("../README.md")]
+
+//! This crate is used for updating structures of [`bdk_chain`] with data from an Esplora server.
+//!
+//! The two primary methods are [`EsploraExt::sync`] and [`EsploraExt::full_scan`]. In most cases
+//! [`EsploraExt::sync`] is used to sync the transaction histories of scripts that the application
+//! cares about, for example the scripts for all the receive addresses of a Wallet's keychain that it
+//! has shown a user. [`EsploraExt::full_scan`] is meant to be used when importing or restoring a
+//! keychain where the range of possibly used scripts is not known. In this case it is necessary to
+//! scan all keychain scripts until a number (the "stop gap") of unused scripts is discovered. For a
+//! sync or full scan the user receives relevant blockchain data and output updates for [`bdk_chain`]
+//! via a new [`TxGraph`] to be appended to any existing [`TxGraph`] data.
+//!
+//! Refer to [`example_esplora`] for a complete example.
+//!
+//! [`TxGraph`]: bdk_chain::tx_graph::TxGraph
+//! [`example_esplora`]: https://github.com/bitcoindevkit/bdk/tree/master/example-crates/example_esplora
+
 use bdk_chain::{BlockId, ConfirmationTimeHeightAnchor};
 use esplora_client::TxStatus;
 

crates/esplora/tests/async_ext.rs

@@ -101,7 +101,7 @@ pub async fn test_update_tx_graph_without_keychain() -> anyhow::Result<()> {
 
     let graph_update = env
         .client
-        .scan_txs(
+        .sync(
             misc_spks.into_iter(),
             vec![].into_iter(),
             vec![].into_iter(),
@@ -166,28 +166,10 @@ pub async fn test_async_update_tx_graph_gap_limit() -> anyhow::Result<()> {
 
     // A scan with a gap limit of 2 won't find the transaction, but a scan with a gap limit of 3
     // will.
-    let (graph_update, active_indices) = env
-        .client
-        .scan_txs_with_keychains(
-            keychains.clone(),
-            vec![].into_iter(),
-            vec![].into_iter(),
-            2,
-            1,
-        )
-        .await?;
+    let (graph_update, active_indices) = env.client.full_scan(keychains.clone(), 2, 1).await?;
     assert!(graph_update.full_txs().next().is_none());
     assert!(active_indices.is_empty());
-    let (graph_update, active_indices) = env
-        .client
-        .scan_txs_with_keychains(
-            keychains.clone(),
-            vec![].into_iter(),
-            vec![].into_iter(),
-            3,
-            1,
-        )
-        .await?;
+    let (graph_update, active_indices) = env.client.full_scan(keychains.clone(), 3, 1).await?;
     assert_eq!(graph_update.full_txs().next().unwrap().txid, txid_4th_addr);
     assert_eq!(active_indices[&0], 3);
 
@@ -209,24 +191,12 @@ pub async fn test_async_update_tx_graph_gap_limit() -> anyhow::Result<()> {
 
     // A scan with gap limit 4 won't find the second transaction, but a scan with gap limit 5 will.
     // The last active indice won't be updated in the first case but will in the second one.
-    let (graph_update, active_indices) = env
-        .client
-        .scan_txs_with_keychains(
-            keychains.clone(),
-            vec![].into_iter(),
-            vec![].into_iter(),
-            4,
-            1,
-        )
-        .await?;
+    let (graph_update, active_indices) = env.client.full_scan(keychains.clone(), 4, 1).await?;
     let txs: HashSet<_> = graph_update.full_txs().map(|tx| tx.txid).collect();
     assert_eq!(txs.len(), 1);
     assert!(txs.contains(&txid_4th_addr));
     assert_eq!(active_indices[&0], 3);
-    let (graph_update, active_indices) = env
-        .client
-        .scan_txs_with_keychains(keychains, vec![].into_iter(), vec![].into_iter(), 5, 1)
-        .await?;
+    let (graph_update, active_indices) = env.client.full_scan(keychains, 5, 1).await?;
     let txs: HashSet<_> = graph_update.full_txs().map(|tx| tx.txid).collect();
     assert_eq!(txs.len(), 2);
     assert!(txs.contains(&txid_4th_addr) && txs.contains(&txid_last_addr));

crates/esplora/tests/blocking_ext.rs

@@ -99,7 +99,7 @@ pub fn test_update_tx_graph_without_keychain() -> anyhow::Result<()> {
         sleep(Duration::from_millis(10))
     }
 
-    let graph_update = env.client.scan_txs(
+    let graph_update = env.client.sync(
         misc_spks.into_iter(),
         vec![].into_iter(),
         vec![].into_iter(),
@@ -164,22 +164,10 @@ pub fn test_update_tx_graph_gap_limit() -> anyhow::Result<()> {
 
     // A scan with a gap limit of 2 won't find the transaction, but a scan with a gap limit of 3
     // will.
-    let (graph_update, active_indices) = env.client.scan_txs_with_keychains(
-        keychains.clone(),
-        vec![].into_iter(),
-        vec![].into_iter(),
-        2,
-        1,
-    )?;
+    let (graph_update, active_indices) = env.client.full_scan(keychains.clone(), 2, 1)?;
     assert!(graph_update.full_txs().next().is_none());
     assert!(active_indices.is_empty());
-    let (graph_update, active_indices) = env.client.scan_txs_with_keychains(
-        keychains.clone(),
-        vec![].into_iter(),
-        vec![].into_iter(),
-        3,
-        1,
-    )?;
+    let (graph_update, active_indices) = env.client.full_scan(keychains.clone(), 3, 1)?;
     assert_eq!(graph_update.full_txs().next().unwrap().txid, txid_4th_addr);
     assert_eq!(active_indices[&0], 3);
 
@@ -201,24 +189,12 @@ pub fn test_update_tx_graph_gap_limit() -> anyhow::Result<()> {
 
     // A scan with gap limit 4 won't find the second transaction, but a scan with gap limit 5 will.
     // The last active indice won't be updated in the first case but will in the second one.
-    let (graph_update, active_indices) = env.client.scan_txs_with_keychains(
-        keychains.clone(),
-        vec![].into_iter(),
-        vec![].into_iter(),
-        4,
-        1,
-    )?;
+    let (graph_update, active_indices) = env.client.full_scan(keychains.clone(), 4, 1)?;
     let txs: HashSet<_> = graph_update.full_txs().map(|tx| tx.txid).collect();
     assert_eq!(txs.len(), 1);
     assert!(txs.contains(&txid_4th_addr));
     assert_eq!(active_indices[&0], 3);
-    let (graph_update, active_indices) = env.client.scan_txs_with_keychains(
-        keychains,
-        vec![].into_iter(),
-        vec![].into_iter(),
-        5,
-        1,
-    )?;
+    let (graph_update, active_indices) = env.client.full_scan(keychains, 5, 1)?;
     let txs: HashSet<_> = graph_update.full_txs().map(|tx| tx.txid).collect();
     assert_eq!(txs.len(), 2);
     assert!(txs.contains(&txid_4th_addr) && txs.contains(&txid_last_addr));

crates/file_store/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "bdk_file_store"
-version = "0.2.0"
+version = "0.4.0"
 edition = "2021"
 license = "MIT OR Apache-2.0"
 repository = "https://github.com/bitcoindevkit/bdk"
@@ -11,7 +11,7 @@ authors = ["Bitcoin Dev Kit Developers"]
 readme = "README.md"
 
 [dependencies]
-bdk_chain = { path = "../chain", version = "0.6.0", features = [ "serde", "miniscript" ] }
+bdk_chain = { path = "../chain", version = "0.8.0", features = [ "serde", "miniscript" ] }
 bincode = { version = "1" }
 serde = { version = "1", features = ["derive"] }
 

crates/file_store/src/store.rs

@@ -105,7 +105,7 @@ where
         })
     }
 
-    /// Attempt to open existing [`Store`] file; create it if the file is non-existant.
+    /// Attempt to open existing [`Store`] file; create it if the file is non-existent.
     ///
     /// Internally, this calls either [`open`] or [`create_new`].
     ///

crates/hwi/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "bdk_hwi"
+version = "0.1.0"
+edition = "2021"
+homepage = "https://bitcoindevkit.org"
+repository = "https://github.com/bitcoindevkit/bdk"
+description = "Utilities to use bdk with hardware wallets"
+license = "MIT OR Apache-2.0"
+readme = "README.md"
+
+[dependencies]
+bdk = { path = "../bdk" }
+hwi = { version = "0.7.0", features = [ "miniscript"] }

crates/hwi/src/lib.rs

@@ -0,0 +1,42 @@
+//! HWI Signer
+//!
+//! This crate contains HWISigner, an implementation of a [`TransactionSigner`] to be
+//! used with hardware wallets.
+//! ```no_run
+//! # use bdk::bitcoin::Network;
+//! # use bdk::signer::SignerOrdering;
+//! # use bdk_hwi::HWISigner;
+//! # use bdk::wallet::AddressIndex::New;
+//! # use bdk::{FeeRate, KeychainKind, SignOptions, Wallet};
+//! # use hwi::HWIClient;
+//! # use std::sync::Arc;
+//! #
+//! # fn main() -> Result<(), Box<dyn std::error::Error>> {
+//! let mut devices = HWIClient::enumerate()?;
+//! if devices.is_empty() {
+//!     panic!("No devices found!");
+//! }
+//! let first_device = devices.remove(0)?;
+//! let custom_signer = HWISigner::from_device(&first_device, Network::Testnet.into())?;
+//!
+//! # let mut wallet = Wallet::new_no_persist(
+//! #     "",
+//! #     None,
+//! #     Network::Testnet,
+//! # )?;
+//! #
+//! // Adding the hardware signer to the BDK wallet
+//! wallet.add_signer(
+//!     KeychainKind::External,
+//!     SignerOrdering(200),
+//!     Arc::new(custom_signer),
+//! );
+//!
+//! # Ok(())
+//! # }
+//! ```
+//!
+//! [`TransactionSigner`]: bdk::wallet::signer::TransactionSigner
+
+mod signer;
+pub use signer::*;

crates/hwi/src/signer.rs

@@ -0,0 +1,94 @@
+use bdk::bitcoin::bip32::Fingerprint;
+use bdk::bitcoin::psbt::PartiallySignedTransaction;
+use bdk::bitcoin::secp256k1::{All, Secp256k1};
+
+use hwi::error::Error;
+use hwi::types::{HWIChain, HWIDevice};
+use hwi::HWIClient;
+
+use bdk::signer::{SignerCommon, SignerError, SignerId, TransactionSigner};
+
+#[derive(Debug)]
+/// Custom signer for Hardware Wallets
+///
+/// This ignores `sign_options` and leaves the decisions up to the hardware wallet.
+pub struct HWISigner {
+    fingerprint: Fingerprint,
+    client: HWIClient,
+}
+
+impl HWISigner {
+    /// Create a instance from the specified device and chain
+    pub fn from_device(device: &HWIDevice, chain: HWIChain) -> Result<HWISigner, Error> {
+        let client = HWIClient::get_client(device, false, chain)?;
+        Ok(HWISigner {
+            fingerprint: device.fingerprint,
+            client,
+        })
+    }
+}
+
+impl SignerCommon for HWISigner {
+    fn id(&self, _secp: &Secp256k1<All>) -> SignerId {
+        SignerId::Fingerprint(self.fingerprint)
+    }
+}
+
+impl TransactionSigner for HWISigner {
+    fn sign_transaction(
+        &self,
+        psbt: &mut PartiallySignedTransaction,
+        _sign_options: &bdk::SignOptions,
+        _secp: &Secp256k1<All>,
+    ) -> Result<(), SignerError> {
+        psbt.combine(
+            self.client
+                .sign_tx(psbt)
+                .map_err(|e| {
+                    SignerError::External(format!("While signing with hardware wallet: {}", e))
+                })?
+                .psbt,
+        )
+        .expect("Failed to combine HW signed psbt with passed PSBT");
+        Ok(())
+    }
+}
+
+// TODO: re-enable this once we have the `get_funded_wallet` test util
+// #[cfg(test)]
+// mod tests {
+//     #[test]
+//     fn test_hardware_signer() {
+//         use std::sync::Arc;
+//
+//         use bdk::tests::get_funded_wallet;
+//         use bdk::signer::SignerOrdering;
+//         use bdk::bitcoin::Network;
+//         use crate::HWISigner;
+//         use hwi::HWIClient;
+//
+//         let mut devices = HWIClient::enumerate().unwrap();
+//         if devices.is_empty() {
+//             panic!("No devices found!");
+//         }
+//         let device = devices.remove(0).unwrap();
+//         let client = HWIClient::get_client(&device, true, Network::Regtest.into()).unwrap();
+//         let descriptors = client.get_descriptors::<String>(None).unwrap();
+//         let custom_signer = HWISigner::from_device(&device, Network::Regtest.into()).unwrap();
+//
+//         let (mut wallet, _) = get_funded_wallet(&descriptors.internal[0]);
+//         wallet.add_signer(
+//             bdk::KeychainKind::External,
+//             SignerOrdering(200),
+//             Arc::new(custom_signer),
+//         );
+//
+//         let addr = wallet.get_address(bdk::wallet::AddressIndex::LastUnused);
+//         let mut builder = wallet.build_tx();
+//         builder.drain_to(addr.script_pubkey()).drain_wallet();
+//         let (mut psbt, _) = builder.finish().unwrap();
+//
+//         let finalized = wallet.sign(&mut psbt, Default::default()).unwrap();
+//         assert!(finalized);
+//     }
+// }

example-crates/example_bitcoind_rpc_polling/src/main.rs

@@ -12,7 +12,7 @@ use bdk_bitcoind_rpc::{
     Emitter,
 };
 use bdk_chain::{
-    bitcoin::{Block, Transaction},
+    bitcoin::{constants::genesis_block, Block, Transaction},
     indexed_tx_graph, keychain,
     local_chain::{self, CheckPoint, LocalChain},
     ConfirmationTimeHeightAnchor, IndexedTxGraph,
@@ -64,9 +64,6 @@ struct RpcArgs {
     /// Starting block height to fallback to if no point of agreement if found
     #[clap(env = "FALLBACK_HEIGHT", long, default_value = "0")]
     fallback_height: u32,
-    /// The unused-scripts lookahead will be kept at this size
-    #[clap(long, default_value = "10")]
-    lookahead: u32,
 }
 
 impl From<RpcArgs> for Auth {
@@ -120,10 +117,11 @@ fn main() -> anyhow::Result<()> {
         "[{:>10}s] loaded initial changeset from db",
         start.elapsed().as_secs_f32()
     );
+    let (init_chain_changeset, init_graph_changeset) = init_changeset;
 
     let graph = Mutex::new({
         let mut graph = IndexedTxGraph::new(index);
-        graph.apply_changeset(init_changeset.1);
+        graph.apply_changeset(init_graph_changeset);
         graph
     });
     println!(
@@ -131,7 +129,16 @@ fn main() -> anyhow::Result<()> {
         start.elapsed().as_secs_f32()
     );
 
-    let chain = Mutex::new(LocalChain::from_changeset(init_changeset.0)?);
+    let chain = Mutex::new(if init_chain_changeset.is_empty() {
+        let genesis_hash = genesis_block(args.network).block_hash();
+        let (chain, chain_changeset) = LocalChain::from_genesis_hash(genesis_hash);
+        let mut db = db.lock().unwrap();
+        db.stage((chain_changeset, Default::default()));
+        db.commit()?;
+        chain
+    } else {
+        LocalChain::from_changeset(init_chain_changeset)?
+    });
     println!(
         "[{:>10}s] loaded local chain from changeset",
         start.elapsed().as_secs_f32()
@@ -161,13 +168,9 @@ fn main() -> anyhow::Result<()> {
     match rpc_cmd {
         RpcCommands::Sync { rpc_args } => {
             let RpcArgs {
-                fallback_height,
-                lookahead,
-                ..
+                fallback_height, ..
             } = rpc_args;
 
-            graph.lock().unwrap().index.set_lookahead_for_all(lookahead);
-
             let chain_tip = chain.lock().unwrap().tip();
             let rpc_client = rpc_args.new_client()?;
             let mut emitter = Emitter::new(&rpc_client, chain_tip, fallback_height);
@@ -233,13 +236,10 @@ fn main() -> anyhow::Result<()> {
         }
         RpcCommands::Live { rpc_args } => {
             let RpcArgs {
-                fallback_height,
-                lookahead,
-                ..
+                fallback_height, ..
             } = rpc_args;
             let sigterm_flag = start_ctrlc_handler();
 
-            graph.lock().unwrap().index.set_lookahead_for_all(lookahead);
             let last_cp = chain.lock().unwrap().tip();
 
             println!(

example-crates/example_cli/src/lib.rs

@@ -478,14 +478,14 @@ where
                         true => Keychain::Internal,
                         false => Keychain::External,
                     };
-                    for (spk_i, spk) in index.revealed_spks_of_keychain(&target_keychain) {
+                    for (spk_i, spk) in index.revealed_keychain_spks(&target_keychain) {
                         let address = Address::from_script(spk, network)
                             .expect("should always be able to derive address");
                         println!(
                             "{:?} {} used:{}",
                             spk_i,
                             address,
-                            index.is_used(&(target_keychain, spk_i))
+                            index.is_used(target_keychain, spk_i)
                         );
                     }
                     Ok(())
@@ -611,7 +611,7 @@ where
                     // We don't want other callers/threads to use this address while we're using it
                     // but we also don't want to scan the tx we just created because it's not
                     // technically in the blockchain yet.
-                    graph.index.mark_used(&change_keychain, index);
+                    graph.index.mark_used(change_keychain, index);
                     (tx, Some((change_keychain, index)))
                 } else {
                     (tx, None)
@@ -636,7 +636,7 @@ where
                 Err(e) => {
                     if let Some((keychain, index)) = change_index {
                         // We failed to broadcast, so allow our change address to be used in the future
-                        graph.lock().unwrap().index.unmark_used(&keychain, index);
+                        graph.lock().unwrap().index.unmark_used(keychain, index);
                     }
                     Err(e)
                 }

example-crates/example_electrum/src/main.rs

@@ -5,7 +5,7 @@ use std::{
 };
 
 use bdk_chain::{
-    bitcoin::{Address, Network, OutPoint, ScriptBuf, Txid},
+    bitcoin::{constants::genesis_block, Address, Network, OutPoint, Txid},
     indexed_tx_graph::{self, IndexedTxGraph},
     keychain,
     local_chain::{self, LocalChain},
@@ -112,7 +112,12 @@ fn main() -> anyhow::Result<()> {
         graph
     });
 
-    let chain = Mutex::new(LocalChain::from_changeset(disk_local_chain)?);
+    let chain = Mutex::new({
+        let genesis_hash = genesis_block(args.network).block_hash();
+        let (mut chain, _) = LocalChain::from_genesis_hash(genesis_hash);
+        chain.apply_changeset(&disk_local_chain)?;
+        chain
+    });
 
     let electrum_cmd = match &args.command {
         example_cli::Commands::ChainSpecific(electrum_cmd) => electrum_cmd,
@@ -150,7 +155,7 @@ fn main() -> anyhow::Result<()> {
 
                 let keychain_spks = graph
                     .index
-                    .spks_of_all_keychains()
+                    .all_unbounded_spk_iters()
                     .into_iter()
                     .map(|(keychain, iter)| {
                         let mut first = true;
@@ -172,14 +177,7 @@ fn main() -> anyhow::Result<()> {
             };
 
             client
-                .scan(
-                    tip,
-                    keychain_spks,
-                    core::iter::empty(),
-                    core::iter::empty(),
-                    stop_gap,
-                    scan_options.batch_size,
-                )
+                .full_scan(tip, keychain_spks, stop_gap, scan_options.batch_size)
                 .context("scanning the blockchain")?
         }
         ElectrumCommands::Sync {
@@ -208,29 +206,28 @@ fn main() -> anyhow::Result<()> {
             if all_spks {
                 let all_spks = graph
                     .index
-                    .all_spks()
-                    .iter()
-                    .map(|(k, v)| (*k, v.clone()))
+                    .revealed_spks()
+                    .map(|(k, i, spk)| (k, i, spk.to_owned()))
                     .collect::<Vec<_>>();
-                spks = Box::new(spks.chain(all_spks.into_iter().map(|(index, script)| {
-                    eprintln!("scanning {:?}", index);
-                    script
+                spks = Box::new(spks.chain(all_spks.into_iter().map(|(k, i, spk)| {
+                    eprintln!("scanning {}:{}", k, i);
+                    spk
                 })));
             }
             if unused_spks {
                 let unused_spks = graph
                     .index
-                    .unused_spks(..)
-                    .map(|(k, v)| (*k, ScriptBuf::from(v)))
+                    .unused_spks()
+                    .map(|(k, i, spk)| (k, i, spk.to_owned()))
                     .collect::<Vec<_>>();
-                spks = Box::new(spks.chain(unused_spks.into_iter().map(|(index, script)| {
+                spks = Box::new(spks.chain(unused_spks.into_iter().map(|(k, i, spk)| {
                     eprintln!(
-                        "Checking if address {} {:?} has been used",
-                        Address::from_script(&script, args.network).unwrap(),
-                        index
+                        "Checking if address {} {}:{} has been used",
+                        Address::from_script(&spk, args.network).unwrap(),
+                        k,
+                        i,
                     );
-
-                    script
+                    spk
                 })));
             }
 
@@ -279,7 +276,7 @@ fn main() -> anyhow::Result<()> {
             drop((graph, chain));
 
             let electrum_update = client
-                .scan_without_keychain(tip, spks, txids, outpoints, scan_options.batch_size)
+                .sync(tip, spks, txids, outpoints, scan_options.batch_size)
                 .context("scanning the blockchain")?;
             (electrum_update, BTreeMap::new())
         }

example-crates/example_esplora/src/main.rs

@@ -165,7 +165,7 @@ fn main() -> anyhow::Result<()> {
                 .lock()
                 .expect("mutex must not be poisoned")
                 .index
-                .spks_of_all_keychains()
+                .all_unbounded_spk_iters()
                 .into_iter()
                 // This `map` is purely for logging.
                 .map(|(keychain, iter)| {
@@ -188,13 +188,7 @@ fn main() -> anyhow::Result<()> {
             // represents the last active spk derivation indices of keychains
             // (`keychain_indices_update`).
             let (graph_update, last_active_indices) = client
-                .scan_txs_with_keychains(
-                    keychain_spks,
-                    core::iter::empty(),
-                    core::iter::empty(),
-                    *stop_gap,
-                    scan_options.parallel_requests,
-                )
+                .full_scan(keychain_spks, *stop_gap, scan_options.parallel_requests)
                 .context("scanning for transactions")?;
 
             let mut graph = graph.lock().expect("mutex must not be poisoned");
@@ -241,32 +235,32 @@ fn main() -> anyhow::Result<()> {
                 if *all_spks {
                     let all_spks = graph
                         .index
-                        .all_spks()
-                        .iter()
-                        .map(|(k, v)| (*k, v.clone()))
+                        .revealed_spks()
+                        .map(|(k, i, spk)| (k, i, spk.to_owned()))
                         .collect::<Vec<_>>();
-                    spks = Box::new(spks.chain(all_spks.into_iter().map(|(index, script)| {
-                        eprintln!("scanning {:?}", index);
+                    spks = Box::new(spks.chain(all_spks.into_iter().map(|(k, i, spk)| {
+                        eprintln!("scanning {}:{}", k, i);
                         // Flush early to ensure we print at every iteration.
                         let _ = io::stderr().flush();
-                        script
+                        spk
                     })));
                 }
                 if unused_spks {
                     let unused_spks = graph
                         .index
-                        .unused_spks(..)
-                        .map(|(k, v)| (*k, v.to_owned()))
+                        .unused_spks()
+                        .map(|(k, i, spk)| (k, i, spk.to_owned()))
                         .collect::<Vec<_>>();
-                    spks = Box::new(spks.chain(unused_spks.into_iter().map(|(index, script)| {
+                    spks = Box::new(spks.chain(unused_spks.into_iter().map(|(k, i, spk)| {
                         eprintln!(
-                            "Checking if address {} {:?} has been used",
-                            Address::from_script(&script, args.network).unwrap(),
-                            index
+                            "Checking if address {} {}:{} has been used",
+                            Address::from_script(&spk, args.network).unwrap(),
+                            k,
+                            i,
                         );
                         // Flush early to ensure we print at every iteration.
                         let _ = io::stderr().flush();
-                        script
+                        spk
                     })));
                 }
                 if utxos {
@@ -312,7 +306,7 @@ fn main() -> anyhow::Result<()> {
             }
 
             let graph_update =
-                client.scan_txs(spks, txids, outpoints, scan_options.parallel_requests)?;
+                client.sync(spks, txids, outpoints, scan_options.parallel_requests)?;
 
             graph.lock().unwrap().apply_update(graph_update)
         }

example-crates/wallet_electrum/src/main.rs

@@ -40,7 +40,7 @@ fn main() -> Result<(), anyhow::Error> {
 
     let prev_tip = wallet.latest_checkpoint();
     let keychain_spks = wallet
-        .spks_of_all_keychains()
+        .all_unbounded_spk_iters()
         .into_iter()
         .map(|(k, k_spks)| {
             let mut once = Some(());
@@ -61,7 +61,7 @@ fn main() -> Result<(), anyhow::Error> {
             relevant_txids,
         },
         keychain_update,
-    ) = client.scan(prev_tip, keychain_spks, None, None, STOP_GAP, BATCH_SIZE)?;
+    ) = client.full_scan(prev_tip, keychain_spks, STOP_GAP, BATCH_SIZE)?;
 
     println!();
 

example-crates/wallet_esplora_async/src/main.rs

@@ -39,7 +39,7 @@ async fn main() -> Result<(), anyhow::Error> {
 
     let prev_tip = wallet.latest_checkpoint();
     let keychain_spks = wallet
-        .spks_of_all_keychains()
+        .all_unbounded_spk_iters()
         .into_iter()
         .map(|(k, k_spks)| {
             let mut once = Some(());
@@ -54,7 +54,7 @@ async fn main() -> Result<(), anyhow::Error> {
         })
         .collect();
     let (update_graph, last_active_indices) = client
-        .scan_txs_with_keychains(keychain_spks, None, None, STOP_GAP, PARALLEL_REQUESTS)
+        .full_scan(keychain_spks, STOP_GAP, PARALLEL_REQUESTS)
         .await?;
     let missing_heights = update_graph.missing_heights(wallet.local_chain());
     let chain_update = client.update_local_chain(prev_tip, missing_heights).await?;

example-crates/wallet_esplora_blocking/src/main.rs

@@ -38,7 +38,7 @@ fn main() -> Result<(), anyhow::Error> {
 
     let prev_tip = wallet.latest_checkpoint();
     let keychain_spks = wallet
-        .spks_of_all_keychains()
+        .all_unbounded_spk_iters()
         .into_iter()
         .map(|(k, k_spks)| {
             let mut once = Some(());
@@ -54,7 +54,7 @@ fn main() -> Result<(), anyhow::Error> {
         .collect();
 
     let (update_graph, last_active_indices) =
-        client.scan_txs_with_keychains(keychain_spks, None, None, STOP_GAP, PARALLEL_REQUESTS)?;
+        client.full_scan(keychain_spks, STOP_GAP, PARALLEL_REQUESTS)?;
     let missing_heights = update_graph.missing_heights(wallet.local_chain());
     let chain_update = client.update_local_chain(prev_tip, missing_heights)?;
     let update = Update {