chore(deps): bump actions/checkout from 1 to 4

Bumps [actions/checkout](https://github.com/actions/checkout) from 1 to 4.
- [Release notes](https://github.com/actions/checkout/releases)
- [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md)
- [Commits](https://github.com/actions/checkout/compare/v1...v4)

---
updated-dependencies:
- dependency-name: actions/checkout
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>

.github/dependabot.yml

@@ -0,0 +1,8 @@
+# Set update schedule for GitHub Actions
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      # Check for updates to GitHub Actions every week
+      interval: "weekly"

.github/workflows/audit.yml

@@ -13,7 +13,7 @@ jobs:
   security_audit:
     runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - uses: actions-rs/audit-check@v1
         with:
           token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/code_coverage.yml

@@ -13,13 +13,13 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: Install lcov tools
         run: sudo apt-get install lcov -y
       - name: Install Rust toolchain
         uses: actions-rs/toolchain@v1
         with:
-            toolchain: "1.65.0"
+            toolchain: stable
             override: true
             profile: minimal
             components: llvm-tools-preview
@@ -27,6 +27,7 @@ jobs:
         uses: Swatinem/rust-cache@v2.2.1
       - name: Install grcov
         run: if [[ ! -e ~/.cargo/bin/grcov ]]; then cargo install grcov; fi
+        # TODO: re-enable the hwi tests
       - name: Build simulator image
         run: docker build -t hwi/ledger_emulator ./ci -f ci/Dockerfile.ledger
       - name: Run simulator image

.github/workflows/cont_integration.yml

@@ -12,13 +12,13 @@ jobs:
         rust:
           - version: stable
             clippy: true
-          - version: 1.57.0 # MSRV
+          - version: 1.63.0 # MSRV
         features:
           - --no-default-features
           - --all-features
     steps:
       - name: checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: Install Rust toolchain
         uses: actions-rs/toolchain@v1
         with:
@@ -27,6 +27,12 @@ jobs:
             profile: minimal
       - name: Rust Cache
         uses: Swatinem/rust-cache@v2.2.1
+      - name: Pin dependencies for MSRV
+        if: matrix.rust.version == '1.63.0'
+        run: |
+          cargo update -p zstd-sys --precise "2.0.8+zstd.1.5.5"
+          cargo update -p time --precise "0.3.20"
+          cargo update -p home --precise "0.5.5"
       - name: Build
         run: cargo build ${{ matrix.features }}
       - name: Test
@@ -37,7 +43,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: Install Rust toolchain
         uses: actions-rs/toolchain@v1
         with:
@@ -68,10 +74,9 @@ jobs:
       CFLAGS: -I/usr/include
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         # Install a recent version of clang that supports wasm32
       - run: wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - || exit 1
-      - run: sudo apt-add-repository "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-10 main" || exit 1
       - run: sudo apt-get update || exit 1
       - run: sudo apt-get install -y libclang-common-10-dev clang-10 libc6-dev-i386 || exit 1
       - name: Install Rust toolchain
@@ -95,7 +100,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: Install Rust toolchain
         uses: actions-rs/toolchain@v1
         with:
@@ -109,12 +114,10 @@ jobs:
   clippy_check:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v4
       - uses: actions-rs/toolchain@v1
         with:
-            # we pin clippy instead of using "stable" so that our CI doesn't break
-            # at each new cargo release
-            toolchain: "1.67.0"
+            toolchain: stable
             components: clippy
             override: true
       - name: Rust Cache

.github/workflows/nightly_docs.yml

@@ -8,7 +8,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout sources
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: Set default toolchain
         run: rustup default nightly-2022-12-14
       - name: Set profile
@@ -34,7 +34,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout `bitcoindevkit.org`
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           ssh-key: ${{ secrets.DOCS_PUSH_SSH_KEY }}
           repository: bitcoindevkit/bitcoindevkit.org

.gitignore

@@ -4,3 +4,6 @@ Cargo.lock
 
 *.swp
 .idea
+
+# Example persisted files.
+*.db

CHANGELOG.md

@@ -158,7 +158,7 @@ BDK and LDK together.
 - Add the ability to specify which leaves to sign in a taproot transaction through `TapLeavesOptions` in `SignOptions`
 - Add the ability to specify whether a taproot transaction should be signed using the internal key or not, using `sign_with_tap_internal_key` in `SignOptions`
 - Consolidate params `fee_amount` and `amount_needed` in `target_amount` in `CoinSelectionAlgorithm::coin_select` signature.
-- Change the meaning of the `fee_amount` field inside `CoinSelectionResult`: from now on the `fee_amount` will represent only the fees asociated with the utxos in the `selected` field of `CoinSelectionResult`.
+- Change the meaning of the `fee_amount` field inside `CoinSelectionResult`: from now on the `fee_amount` will represent only the fees associated with the utxos in the `selected` field of `CoinSelectionResult`.
 - New `RpcBlockchain` implementation with various fixes.
 - Return balance in separate categories, namely `confirmed`, `trusted_pending`, `untrusted_pending` & `immature`.
 
@@ -449,7 +449,7 @@ final transaction is created by calling `finish` on the builder.
 #### Changed
 - Simplify the architecture of blockchain traits
 - Improve sync
-- Remove unused varaint HeaderParseFail
+- Remove unused variant `HeaderParseFail`
 
 ### CLI
 #### Added
@@ -517,7 +517,7 @@ final transaction is created by calling `finish` on the builder.
 - Default to SIGHASH_ALL if not specified
 - Replace ChangeSpendPolicy::filter_utxos with a predicate
 - Make 'unspendable' into a HashSet
-- Stop implicitly enforcing manaul selection by .add_utxo
+- Stop implicitly enforcing manual selection by .add_utxo
 - Rename DumbCS to LargestFirstCoinSelection
 - Rename must_use_utxos to required_utxos
 - Rename may_use_utxos to optional_uxtos
@@ -529,7 +529,7 @@ final transaction is created by calling `finish` on the builder.
 - Use TXIN_DEFAULT_WEIGHT constant in coin selection
 - Replace `must_use` with `required` in coin selection
 - Take both spending policies into account in create_tx
-- Check last derivation in cache to avoid recomputation
+- Check last derivation in cache to avoid recomputing
 - Use the branch-and-bound cs by default
 - Make coin_select return UTXOs instead of TxIns
 - Build output lookup inside complete transaction
@@ -550,7 +550,7 @@ final transaction is created by calling `finish` on the builder.
 - Require esplora feature for repl example
 
 #### Security
-- Use dirs-next instead of dirs since the latter is unmantained
+- Use dirs-next instead of dirs since the latter is unmaintained
 
 ## [0.1.0-beta.1] - 2020-09-08
 

CONTRIBUTING.md

@@ -46,15 +46,15 @@ Every new feature should be covered by functional tests where possible.
 When refactoring, structure your PR to make it easy to review and don't
 hesitate to split it into multiple small, focused PRs.
 
-The Minimal Supported Rust Version is 1.46 (enforced by our CI).
+The Minimal Supported Rust Version is **1.57.0** (enforced by our CI).
 
 Commits should cover both the issue fixed and the solution's rationale.
-These [guidelines](https://chris.beams.io/posts/git-commit/) should be kept in mind.
+These [guidelines](https://chris.beams.io/posts/git-commit/) should be kept in mind. Commit messages should follow the ["Conventional Commits 1.0.0"](https://www.conventionalcommits.org/en/v1.0.0/) to make commit histories easier to read by humans and automated tools. 
 
 To facilitate communication with other contributors, the project is making use
 of GitHub's "assignee" field. First check that no one is assigned and then
 comment suggesting that you're working on it. If someone is already assigned,
-don't hesitate to ask if the assigned party or previous commenters are still
+don't hesitate to ask if the assigned party or previous commenter are still
 working on it if it has been awhile.
 
 Deprecation policy
@@ -91,7 +91,7 @@ This is also enforced by the CI.
 Security
 --------
 
-Security is a high priority of BDK; disclosure of security vulnerabilites helps
+Security is a high priority of BDK; disclosure of security vulnerabilities helps
 prevent user loss of funds.
 
 Note that BDK is currently considered "pre-production" during this time, there

Cargo.toml

@@ -1,15 +1,21 @@
 [workspace]
+resolver = "2"
 members = [
     "crates/bdk",
     "crates/chain",
     "crates/file_store",
     "crates/electrum",
     "crates/esplora",
+    "crates/bitcoind_rpc",
+    "crates/hwi",
     "example-crates/example_cli",
     "example-crates/example_electrum",
+    "example-crates/example_esplora",
+    "example-crates/example_bitcoind_rpc_polling",
     "example-crates/wallet_electrum",
-    "example-crates/wallet_esplora",
+    "example-crates/wallet_esplora_blocking",
     "example-crates/wallet_esplora_async",
+    "example-crates/wallet_rpc",
     "nursery/tmp_plan",
     "nursery/coin_select"
 ]

README.md

@@ -15,7 +15,7 @@
     <a href="https://github.com/bitcoindevkit/bdk/actions?query=workflow%3ACI"><img alt="CI Status" src="https://github.com/bitcoindevkit/bdk/workflows/CI/badge.svg"></a>
     <a href="https://coveralls.io/github/bitcoindevkit/bdk?branch=master"><img src="https://coveralls.io/repos/github/bitcoindevkit/bdk/badge.svg?branch=master"/></a>
     <a href="https://docs.rs/bdk"><img alt="API Docs" src="https://img.shields.io/badge/docs.rs-bdk-green"/></a>
-    <a href="https://blog.rust-lang.org/2021/12/02/Rust-1.57.0.html"><img alt="Rustc Version 1.57.0+" src="https://img.shields.io/badge/rustc-1.57.0%2B-lightgrey.svg"/></a>
+    <a href="https://blog.rust-lang.org/2022/08/11/Rust-1.63.0.html"><img alt="Rustc Version 1.63.0+" src="https://img.shields.io/badge/rustc-1.63.0%2B-lightgrey.svg"/></a>
     <a href="https://discord.gg/d7NkDKm"><img alt="Chat on Discord" src="https://img.shields.io/discord/753336465005608961?logo=discord"></a>
   </p>
 
@@ -33,7 +33,7 @@ It is built upon the excellent [`rust-bitcoin`] and [`rust-miniscript`] crates.
 
 > ⚠ The Bitcoin Dev Kit developers are in the process of releasing a `v1.0` which is a fundamental re-write of how the library works.
 > See for some background on this project: https://bitcoindevkit.org/blog/road-to-bdk-1/ (ignore the timeline 😁)
-> For a release timeline see the [`bdk_core_staging`] repo where a lot of the component work is being done. The plan is that everything in the `bdk_core_staging` repo will be moved into the `crates` directory here.
+> For a release timeline see the [`BDK 1.0 project page`].
 
 ## Architecture
 
@@ -45,10 +45,50 @@ The project is split up into several crates in the `/crates` directory:
 - [`esplora`](./crates/esplora): Extends the [`esplora-client`] crate with methods to fetch chain data from an esplora HTTP server in the form that [`bdk_chain`] and `Wallet` can consume.
 - [`electrum`](./crates/electrum): Extends the [`electrum-client`] crate with methods to fetch chain data from an electrum server in the form that [`bdk_chain`] and `Wallet` can consume.
 
-Fully working examples of how to use these components are in `/example-crates`
+Fully working examples of how to use these components are in `/example-crates`:
+- [`example_cli`](./example-crates/example_cli): Library used by the `example_*` crates. Provides utilities for syncing, showing the balance, generating addresses and creating transactions without using the bdk `Wallet`.
+- [`example_electrum`](./example-crates/example_electrum): A command line Bitcoin wallet application built on top of `example_cli` and the `electrum` crate. It shows the power of the bdk tools (`chain` + `file_store` + `electrum`), without depending on the main `bdk` library.
+- [`example_esplora`](./example-crates/example_esplora): A command line Bitcoin wallet application built on top of `example_cli` and the `esplora` crate. It shows the power of the bdk tools (`chain` + `file_store` + `esplora`), without depending on the main `bdk` library.
+- [`example_bitcoind_rpc_polling`](./example-crates/example_bitcoind_rpc_polling): A command line Bitcoin wallet application built on top of `example_cli` and the `bitcoind_rpc` crate. It shows the power of the bdk tools (`chain` + `file_store` + `bitcoind_rpc`), without depending on the main `bdk` library.
+- [`wallet_esplora_blocking`](./example-crates/wallet_esplora_blocking): Uses the `Wallet` to sync and spend using the Esplora blocking interface.
+- [`wallet_esplora_async`](./example-crates/wallet_esplora_async): Uses the `Wallet` to sync and spend using the Esplora asynchronous interface.
+- [`wallet_electrum`](./example-crates/wallet_electrum): Uses the `Wallet` to sync and spend using Electrum.
 
-[`bdk_core_staging`]: https://github.com/LLFourn/bdk_core_staging
+[`BDK 1.0 project page`]: https://github.com/orgs/bitcoindevkit/projects/14
 [`rust-miniscript`]: https://github.com/rust-bitcoin/rust-miniscript
 [`rust-bitcoin`]: https://github.com/rust-bitcoin/rust-bitcoin
-[`esplora-client`]: https://docs.rs/esplora-client/0.3.0/esplora_client/
-[`electrum-client`]: https://docs.rs/electrum-client/0.13.0/electrum_client/
+[`esplora-client`]: https://docs.rs/esplora-client/
+[`electrum-client`]: https://docs.rs/electrum-client/
+[`bdk_chain`]: https://docs.rs/bdk-chain/
+
+## Minimum Supported Rust Version (MSRV)
+This library should compile with any combination of features with Rust 1.63.0.
+
+To build with the MSRV you will need to pin dependencies as follows:
+
+```shell
+# zip 0.6.3 has MSRV 1.64.0
+cargo update -p zip --precise "0.6.2"
+# time 0.3.21 has MSRV 1.65.0
+cargo update -p time --precise "0.3.20"
+# jobserver 0.1.27 has MSRV 1.66.0
+cargo update -p jobserver --precise "0.1.26"
+# home 0.5.9 has MSRV 1.70.0
+cargo update -p home --precise "0.5.5"
+```
+
+## License
+
+Licensed under either of
+
+* Apache License, Version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or <https://www.apache.org/licenses/LICENSE-2.0>)
+* MIT license ([LICENSE-MIT](LICENSE-MIT) or <https://opensource.org/licenses/MIT>)
+
+at your option.
+
+### Contribution
+
+Unless you explicitly state otherwise, any contribution intentionally
+submitted for inclusion in the work by you, as defined in the Apache-2.0
+license, shall be dual licensed as above, without any additional terms or
+conditions.

clippy.toml

@@ -1 +1 @@
-msrv="1.57.0"
+msrv="1.63.0"

crates/bdk/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "bdk"
 homepage = "https://bitcoindevkit.org"
-version = "1.0.0-alpha.1"
+version = "1.0.0-alpha.7"
 repository = "https://github.com/bitcoindevkit/bdk"
 documentation = "https://docs.rs/bdk"
 description = "A modern, lightweight, descriptor-based wallet library"
@@ -10,20 +10,18 @@ readme = "README.md"
 license = "MIT OR Apache-2.0"
 authors = ["Bitcoin Dev Kit Developers"]
 edition = "2021"
-rust-version = "1.57"
+rust-version = "1.63"
 
 [dependencies]
-log = "=0.4.18"
 rand = "^0.8"
-miniscript = { version = "9", features = ["serde"], default-features = false }
-bitcoin = { version = "0.29", features = ["serde", "base64", "rand"], default-features = false }
+miniscript = { version = "10.0.0", features = ["serde"], default-features = false }
+bitcoin = { version = "0.30.0", features = ["serde", "base64", "rand-std"], default-features = false }
 serde = { version = "^1.0", features = ["derive"] }
 serde_json = { version = "^1.0" }
-bdk_chain = { path = "../chain", version = "0.5.0", features = ["miniscript", "serde"], default-features = false }
+bdk_chain = { path = "../chain", version = "0.11.0", features = ["miniscript", "serde"], default-features = false }
 
 # Optional dependencies
-hwi = { version = "0.5", optional = true, features = [ "use-miniscript"] }
-bip39 = { version = "1.0.1", optional = true }
+bip39 = { version = "2.0", optional = true }
 
 [target.'cfg(target_arch = "wasm32")'.dependencies]
 getrandom = "0.2"
@@ -35,8 +33,6 @@ std = ["bitcoin/std", "miniscript/std", "bdk_chain/std"]
 compiler = ["miniscript/compiler"]
 all-keys = ["keys-bip39"]
 keys-bip39 = ["bip39"]
-hardware-signer = ["hwi"]
-test-hardware-signer = ["hardware-signer"]
 
 # This feature is used to run `cargo check` in our CI targeting wasm. It's not recommended
 # for libraries to explicitly include the "getrandom/js" feature, so we only do it when
@@ -45,10 +41,10 @@ dev-getrandom-wasm = ["getrandom/js"]
 
 [dev-dependencies]
 lazy_static = "1.4"
-env_logger = "0.7"
-# Move back to importing from rust-bitcoin once https://github.com/rust-bitcoin/rust-bitcoin/pull/1342 is released
-base64 = "^0.13"
 assert_matches = "1.5.0"
+tempfile = "3"
+bdk_file_store = { path = "../file_store" }
+anyhow = "1"
 
 [package.metadata.docs.rs]
 all-features = true

crates/bdk/README.md

@@ -13,7 +13,7 @@
     <a href="https://github.com/bitcoindevkit/bdk/actions?query=workflow%3ACI"><img alt="CI Status" src="https://github.com/bitcoindevkit/bdk/workflows/CI/badge.svg"></a>
     <a href="https://coveralls.io/github/bitcoindevkit/bdk?branch=master"><img src="https://coveralls.io/repos/github/bitcoindevkit/bdk/badge.svg?branch=master"/></a>
     <a href="https://docs.rs/bdk"><img alt="API Docs" src="https://img.shields.io/badge/docs.rs-bdk-green"/></a>
-    <a href="https://blog.rust-lang.org/2021/12/02/Rust-1.57.0.html"><img alt="Rustc Version 1.57.0+" src="https://img.shields.io/badge/rustc-1.57.0%2B-lightgrey.svg"/></a>
+    <a href="https://blog.rust-lang.org/2022/08/11/Rust-1.63.0.html"><img alt="Rustc Version 1.63.0+" src="https://img.shields.io/badge/rustc-1.63.0%2B-lightgrey.svg"/></a>
     <a href="https://discord.gg/d7NkDKm"><img alt="Chat on Discord" src="https://img.shields.io/discord/753336465005608961?logo=discord"></a>
   </p>
 
@@ -137,7 +137,7 @@ fn main() {
 <!-- use bdk::electrum_client::Client; -->
 <!-- use bdk::wallet::AddressIndex::New; -->
 
-<!-- use base64; -->
+<!-- use bitcoin::base64; -->
 <!-- use bdk::bitcoin::consensus::serialize; -->
 <!-- use bdk::bitcoin::Network; -->
 
@@ -174,7 +174,7 @@ fn main() {
 <!-- ```rust,no_run -->
 <!-- use bdk::{Wallet, SignOptions}; -->
 
-<!-- use base64; -->
+<!-- use bitcoin::base64; -->
 <!-- use bdk::bitcoin::consensus::deserialize; -->
 <!-- use bdk::bitcoin::Network; -->
 
@@ -206,18 +206,17 @@ cargo test
 
 Licensed under either of
 
- * Apache License, Version 2.0
-   ([LICENSE-APACHE](LICENSE-APACHE) or <https://www.apache.org/licenses/LICENSE-2.0>)
- * MIT license
-   ([LICENSE-MIT](LICENSE-MIT) or <https://opensource.org/licenses/MIT>)
+* Apache License, Version 2.0, ([LICENSE-APACHE](../../LICENSE-APACHE) or <https://www.apache.org/licenses/LICENSE-2.0>)
+* MIT license ([LICENSE-MIT](../../LICENSE-MIT) or <https://opensource.org/licenses/MIT>)
 
 at your option.
 
-## Contribution
+### Contribution
 
-Unless you explicitly state otherwise, any contribution intentionally submitted
-for inclusion in the work by you, as defined in the Apache-2.0 license, shall be
-dual licensed as above, without any additional terms or conditions.
+Unless you explicitly state otherwise, any contribution intentionally
+submitted for inclusion in the work by you, as defined in the Apache-2.0
+license, shall be dual licensed as above, without any additional terms or
+conditions.
 
 [`bdk_chain`]: https://docs.rs/bdk_chain/latest
 [`bdk_file_store`]: https://docs.rs/bdk_file_store/latest

crates/bdk/examples/compiler.rs

@@ -11,15 +11,12 @@
 
 extern crate bdk;
 extern crate bitcoin;
-extern crate log;
 extern crate miniscript;
 extern crate serde_json;
 
 use std::error::Error;
 use std::str::FromStr;
 
-use log::info;
-
 use bitcoin::Network;
 use miniscript::policy::Concrete;
 use miniscript::Descriptor;
@@ -36,13 +33,9 @@ use bdk::{KeychainKind, Wallet};
 /// This example demonstrates the interaction between a bdk wallet and miniscript policy.
 
 fn main() -> Result<(), Box<dyn Error>> {
-    env_logger::init_from_env(
-        env_logger::Env::default().filter_or(env_logger::DEFAULT_FILTER_ENV, "info"),
-    );
-
     // We start with a generic miniscript policy string
     let policy_str = "or(10@thresh(4,pk(029ffbe722b147f3035c87cb1c60b9a5947dd49c774cc31e94773478711a929ac0),pk(025f05815e3a1a8a83bfbb03ce016c9a2ee31066b98f567f6227df1d76ec4bd143),pk(025625f41e4a065efc06d5019cbbd56fe8c07595af1231e7cbc03fafb87ebb71ec),pk(02a27c8b850a00f67da3499b60562673dcf5fdfb82b7e17652a7ac54416812aefd),pk(03e618ec5f384d6e19ca9ebdb8e2119e5bef978285076828ce054e55c4daf473e2)),1@and(older(4209713),thresh(2,pk(03deae92101c790b12653231439f27b8897264125ecb2f46f48278603102573165),pk(033841045a531e1adf9910a6ec279589a90b3b8a904ee64ffd692bd08a8996c1aa),pk(02aebf2d10b040eb936a6f02f44ee82f8b34f5c1ccb20ff3949c2b28206b7c1068))))";
-    info!("Compiling policy: \n{}", policy_str);
+    println!("Compiling policy: \n{}", policy_str);
 
     // Parse the string as a [`Concrete`] type miniscript policy.
     let policy = Concrete::<String>::from_str(policy_str)?;
@@ -51,12 +44,12 @@ fn main() -> Result<(), Box<dyn Error>> {
     // `policy.compile()` returns the resulting miniscript from the policy.
     let descriptor = Descriptor::new_wsh(policy.compile()?)?;
 
-    info!("Compiled into following Descriptor: \n{}", descriptor);
+    println!("Compiled into following Descriptor: \n{}", descriptor);
 
     // Create a new wallet from this descriptor
     let mut wallet = Wallet::new_no_persist(&format!("{}", descriptor), None, Network::Regtest)?;
 
-    info!(
+    println!(
         "First derived address from the descriptor: \n{}",
         wallet.get_address(New)
     );
@@ -64,7 +57,7 @@ fn main() -> Result<(), Box<dyn Error>> {
     // BDK also has it's own `Policy` structure to represent the spending condition in a more
     // human readable json format.
     let spending_policy = wallet.policies(KeychainKind::External)?;
-    info!(
+    println!(
         "The BDK spending policy: \n{}",
         serde_json::to_string_pretty(&spending_policy)?
     );

crates/bdk/examples/mnemonic_to_descriptors.rs

@@ -6,21 +6,20 @@
 // You may not use this file except in accordance with one or both of these
 // licenses.
 
+use anyhow::anyhow;
+use bdk::bitcoin::bip32::DerivationPath;
 use bdk::bitcoin::secp256k1::Secp256k1;
-use bdk::bitcoin::util::bip32::DerivationPath;
 use bdk::bitcoin::Network;
 use bdk::descriptor;
 use bdk::descriptor::IntoWalletDescriptor;
 use bdk::keys::bip39::{Language, Mnemonic, WordCount};
 use bdk::keys::{GeneratableKey, GeneratedKey};
 use bdk::miniscript::Tap;
-use bdk::Error as BDK_Error;
-use std::error::Error;
 use std::str::FromStr;
 
 /// This example demonstrates how to generate a mnemonic phrase
 /// using BDK and use that to generate a descriptor string.
-fn main() -> Result<(), Box<dyn Error>> {
+fn main() -> Result<(), anyhow::Error> {
     let secp = Secp256k1::new();
 
     // In this example we are generating a 12 words mnemonic phrase
@@ -28,14 +27,14 @@ fn main() -> Result<(), Box<dyn Error>> {
     // using their respective `WordCount` variant.
     let mnemonic: GeneratedKey<_, Tap> =
         Mnemonic::generate((WordCount::Words12, Language::English))
-            .map_err(|_| BDK_Error::Generic("Mnemonic generation error".to_string()))?;
+            .map_err(|_| anyhow!("Mnemonic generation error"))?;
 
     println!("Mnemonic phrase: {}", *mnemonic);
     let mnemonic_with_passphrase = (mnemonic, None);
 
     // define external and internal derivation key path
-    let external_path = DerivationPath::from_str("m/86h/0h/0h/0").unwrap();
-    let internal_path = DerivationPath::from_str("m/86h/0h/0h/1").unwrap();
+    let external_path = DerivationPath::from_str("m/86h/1h/0h/0").unwrap();
+    let internal_path = DerivationPath::from_str("m/86h/1h/0h/1").unwrap();
 
     // generate external and internal descriptor from mnemonic
     let (external_descriptor, ext_keymap) =

crates/bdk/examples/policy.rs

@@ -10,8 +10,6 @@
 // licenses.
 
 extern crate bdk;
-extern crate env_logger;
-extern crate log;
 use std::error::Error;
 
 use bdk::bitcoin::Network;
@@ -29,10 +27,6 @@ use bdk::wallet::signer::SignersContainer;
 /// one of the Extend Private key.
 
 fn main() -> Result<(), Box<dyn Error>> {
-    env_logger::init_from_env(
-        env_logger::Env::default().filter_or(env_logger::DEFAULT_FILTER_ENV, "info"),
-    );
-
     let secp = bitcoin::secp256k1::Secp256k1::new();
 
     // The descriptor used in the example
@@ -48,7 +42,7 @@ fn main() -> Result<(), Box<dyn Error>> {
     // But they can be used as independent tools also.
     let (wallet_desc, keymap) = desc.into_wallet_descriptor(&secp, Network::Testnet)?;
 
-    log::info!("Example Descriptor for policy analysis : {}", wallet_desc);
+    println!("Example Descriptor for policy analysis : {}", wallet_desc);
 
     // Create the signer with the keymap and descriptor.
     let signers_container = SignersContainer::build(keymap, &wallet_desc, &secp);
@@ -60,7 +54,7 @@ fn main() -> Result<(), Box<dyn Error>> {
         .extract_policy(&signers_container, BuildSatisfaction::None, &secp)?
         .expect("We expect a policy");
 
-    log::info!("Derived Policy for the descriptor {:#?}", policy);
+    println!("Derived Policy for the descriptor {:#?}", policy);
 
     Ok(())
 }

crates/bdk/src/descriptor/checksum.rs

@@ -42,22 +42,16 @@ fn poly_mod(mut c: u64, val: u64) -> u64 {
     c
 }
 
-/// Computes the checksum bytes of a descriptor.
-/// `exclude_hash = true` ignores all data after the first '#' (inclusive).
-pub(crate) fn calc_checksum_bytes_internal(
-    mut desc: &str,
-    exclude_hash: bool,
-) -> Result<[u8; 8], DescriptorError> {
+/// Compute the checksum bytes of a descriptor, excludes any existing checksum in the descriptor string from the calculation
+pub fn calc_checksum_bytes(mut desc: &str) -> Result<[u8; 8], DescriptorError> {
     let mut c = 1;
     let mut cls = 0;
     let mut clscount = 0;
 
     let mut original_checksum = None;
-    if exclude_hash {
-        if let Some(split) = desc.split_once('#') {
-            desc = split.0;
-            original_checksum = Some(split.1);
-        }
+    if let Some(split) = desc.split_once('#') {
+        desc = split.0;
+        original_checksum = Some(split.1);
     }
 
     for ch in desc.as_bytes() {
@@ -95,39 +89,10 @@ pub(crate) fn calc_checksum_bytes_internal(
     Ok(checksum)
 }
 
-/// Compute the checksum bytes of a descriptor, excludes any existing checksum in the descriptor string from the calculation
-pub fn calc_checksum_bytes(desc: &str) -> Result<[u8; 8], DescriptorError> {
-    calc_checksum_bytes_internal(desc, true)
-}
-
 /// Compute the checksum of a descriptor, excludes any existing checksum in the descriptor string from the calculation
 pub fn calc_checksum(desc: &str) -> Result<String, DescriptorError> {
     // unsafe is okay here as the checksum only uses bytes in `CHECKSUM_CHARSET`
-    calc_checksum_bytes_internal(desc, true)
-        .map(|b| unsafe { String::from_utf8_unchecked(b.to_vec()) })
-}
-
-// TODO in release 0.25.0, remove get_checksum_bytes and get_checksum
-// TODO in release 0.25.0, consolidate calc_checksum_bytes_internal into calc_checksum_bytes
-
-/// Compute the checksum bytes of a descriptor
-#[deprecated(
-    since = "0.24.0",
-    note = "Use new `calc_checksum_bytes` function which excludes any existing checksum in the descriptor string before calculating the checksum hash bytes. See https://github.com/bitcoindevkit/bdk/pull/765."
-)]
-pub fn get_checksum_bytes(desc: &str) -> Result<[u8; 8], DescriptorError> {
-    calc_checksum_bytes_internal(desc, false)
-}
-
-/// Compute the checksum of a descriptor
-#[deprecated(
-    since = "0.24.0",
-    note = "Use new `calc_checksum` function which excludes any existing checksum in the descriptor string before calculating the checksum hash. See https://github.com/bitcoindevkit/bdk/pull/765."
-)]
-pub fn get_checksum(desc: &str) -> Result<String, DescriptorError> {
-    // unsafe is okay here as the checksum only uses bytes in `CHECKSUM_CHARSET`
-    calc_checksum_bytes_internal(desc, false)
-        .map(|b| unsafe { String::from_utf8_unchecked(b.to_vec()) })
+    calc_checksum_bytes(desc).map(|b| unsafe { String::from_utf8_unchecked(b.to_vec()) })
 }
 
 #[cfg(test)]

crates/bdk/src/descriptor/dsl.rs

@@ -516,13 +516,14 @@ macro_rules! descriptor {
         use $crate::miniscript::descriptor::{Descriptor, DescriptorPublicKey};
 
         $crate::impl_top_level_pk!(Pkh, $crate::miniscript::Legacy, $key)
+            .and_then(|(a, b, c)| Ok((a.map_err(|e| miniscript::Error::from(e))?, b, c)))
             .map(|(a, b, c)| (Descriptor::<DescriptorPublicKey>::Pkh(a), b, c))
     });
     ( wpkh ( $key:expr ) ) => ({
         use $crate::miniscript::descriptor::{Descriptor, DescriptorPublicKey};
 
         $crate::impl_top_level_pk!(Wpkh, $crate::miniscript::Segwitv0, $key)
-            .and_then(|(a, b, c)| Ok((a?, b, c)))
+            .and_then(|(a, b, c)| Ok((a.map_err(|e| miniscript::Error::from(e))?, b, c)))
             .map(|(a, b, c)| (Descriptor::<DescriptorPublicKey>::Wpkh(a), b, c))
     });
     ( sh ( wpkh ( $key:expr ) ) ) => ({
@@ -532,7 +533,7 @@ macro_rules! descriptor {
         use $crate::miniscript::descriptor::{Descriptor, DescriptorPublicKey, Sh};
 
         $crate::impl_top_level_pk!(Wpkh, $crate::miniscript::Segwitv0, $key)
-            .and_then(|(a, b, c)| Ok((a?, b, c)))
+            .and_then(|(a, b, c)| Ok((a.map_err(|e| miniscript::Error::from(e))?, b, c)))
             .and_then(|(a, b, c)| Ok((Descriptor::<DescriptorPublicKey>::Sh(Sh::new_wpkh(a.into_inner())?), b, c)))
     });
     ( sh ( $( $minisc:tt )* ) ) => ({
@@ -702,7 +703,7 @@ macro_rules! fragment {
         $crate::keys::make_pkh($key, &secp)
     });
     ( after ( $value:expr ) ) => ({
-        $crate::impl_leaf_opcode_value!(After, $crate::bitcoin::PackedLockTime($value)) // TODO!! https://github.com/rust-bitcoin/rust-bitcoin/issues/1302
+        $crate::impl_leaf_opcode_value!(After, $crate::miniscript::AbsLockTime::from_consensus($value))
     });
     ( older ( $value:expr ) ) => ({
         $crate::impl_leaf_opcode_value!(Older, $crate::bitcoin::Sequence($value)) // TODO!!
@@ -796,7 +797,6 @@ macro_rules! fragment {
 #[cfg(test)]
 mod test {
     use alloc::string::ToString;
-    use bitcoin::hashes::hex::ToHex;
     use bitcoin::secp256k1::Secp256k1;
     use miniscript::descriptor::{DescriptorPublicKey, KeyMap};
     use miniscript::{Descriptor, Legacy, Segwitv0};
@@ -805,8 +805,8 @@ mod test {
 
     use crate::descriptor::{DescriptorError, DescriptorMeta};
     use crate::keys::{DescriptorKey, IntoDescriptorKey, ValidNetworks};
+    use bitcoin::bip32;
     use bitcoin::network::constants::Network::{Bitcoin, Regtest, Signet, Testnet};
-    use bitcoin::util::bip32;
     use bitcoin::PrivateKey;
 
     // test the descriptor!() macro
@@ -822,18 +822,15 @@ mod test {
         assert_eq!(desc.is_witness(), is_witness);
         assert_eq!(!desc.has_wildcard(), is_fixed);
         for i in 0..expected.len() {
-            let index = i as u32;
-            let child_desc = if !desc.has_wildcard() {
-                desc.at_derivation_index(0)
-            } else {
-                desc.at_derivation_index(index)
-            };
+            let child_desc = desc
+                .at_derivation_index(i as u32)
+                .expect("i is not hardened");
             let address = child_desc.address(Regtest);
             if let Ok(address) = address {
                 assert_eq!(address.to_string(), *expected.get(i).unwrap());
             } else {
                 let script = child_desc.script_pubkey();
-                assert_eq!(script.to_hex().as_str(), *expected.get(i).unwrap());
+                assert_eq!(script.to_hex_string(), *expected.get(i).unwrap());
             }
         }
     }
@@ -1178,9 +1175,7 @@ mod test {
     }
 
     #[test]
-    #[should_panic(
-        expected = "Miniscript(ContextError(CompressedOnly(\"04b4632d08485ff1df2db55b9dafd23347d1c47a457072a1e87be26896549a87378ec38ff91d43e8c2092ebda601780485263da089465619e0358a5c1be7ac91f4\")))"
-    )]
+    #[should_panic(expected = "Miniscript(ContextError(UncompressedKeysNotAllowed))")]
     fn test_dsl_miniscript_checks() {
         let mut uncompressed_pk =
             PrivateKey::from_wif("L5EZftvrYaSudiozVRzTqLcHLNDoVn7H5HSfM9BAN6tMJX8oTWz6").unwrap();

crates/bdk/src/descriptor/error.rs

@@ -10,7 +10,6 @@
 // licenses.
 
 //! Descriptor errors
-
 use core::fmt;
 
 /// Errors related to the parsing and usage of descriptors
@@ -22,6 +21,8 @@ pub enum Error {
     InvalidDescriptorChecksum,
     /// The descriptor contains hardened derivation steps on public extended keys
     HardenedDerivationXpub,
+    /// The descriptor contains multipath keys
+    MultiPath,
 
     /// Error thrown while working with [`keys`](crate::keys)
     Key(crate::keys::KeyError),
@@ -32,11 +33,11 @@ pub enum Error {
     InvalidDescriptorCharacter(u8),
 
     /// BIP32 error
-    Bip32(bitcoin::util::bip32::Error),
+    Bip32(bitcoin::bip32::Error),
     /// Error during base58 decoding
-    Base58(bitcoin::util::base58::Error),
+    Base58(bitcoin::base58::Error),
     /// Key-related error
-    Pk(bitcoin::util::key::Error),
+    Pk(bitcoin::key::Error),
     /// Miniscript error
     Miniscript(miniscript::Error),
     /// Hex decoding error
@@ -64,6 +65,10 @@ impl fmt::Display for Error {
                 f,
                 "The descriptor contains hardened derivation steps on public extended keys"
             ),
+            Self::MultiPath => write!(
+                f,
+                "The descriptor contains multipath keys, which are not supported yet"
+            ),
             Self::Key(err) => write!(f, "Key error: {}", err),
             Self::Policy(err) => write!(f, "Policy error: {}", err),
             Self::InvalidDescriptorCharacter(char) => {
@@ -81,9 +86,38 @@ impl fmt::Display for Error {
 #[cfg(feature = "std")]
 impl std::error::Error for Error {}
 
-impl_error!(bitcoin::util::bip32::Error, Bip32);
-impl_error!(bitcoin::util::base58::Error, Base58);
-impl_error!(bitcoin::util::key::Error, Pk);
-impl_error!(miniscript::Error, Miniscript);
-impl_error!(bitcoin::hashes::hex::Error, Hex);
-impl_error!(crate::descriptor::policy::PolicyError, Policy);
+impl From<bitcoin::bip32::Error> for Error {
+    fn from(err: bitcoin::bip32::Error) -> Self {
+        Error::Bip32(err)
+    }
+}
+
+impl From<bitcoin::base58::Error> for Error {
+    fn from(err: bitcoin::base58::Error) -> Self {
+        Error::Base58(err)
+    }
+}
+
+impl From<bitcoin::key::Error> for Error {
+    fn from(err: bitcoin::key::Error) -> Self {
+        Error::Pk(err)
+    }
+}
+
+impl From<miniscript::Error> for Error {
+    fn from(err: miniscript::Error) -> Self {
+        Error::Miniscript(err)
+    }
+}
+
+impl From<bitcoin::hashes::hex::Error> for Error {
+    fn from(err: bitcoin::hashes::hex::Error) -> Self {
+        Error::Hex(err)
+    }
+}
+
+impl From<crate::descriptor::policy::PolicyError> for Error {
+    fn from(err: crate::descriptor::policy::PolicyError) -> Self {
+        Error::Policy(err)
+    }
+}

crates/bdk/src/descriptor/mod.rs

@@ -18,17 +18,17 @@ use crate::collections::BTreeMap;
 use alloc::string::String;
 use alloc::vec::Vec;
 
-use bitcoin::util::bip32::{ChildNumber, DerivationPath, ExtendedPubKey, Fingerprint, KeySource};
-use bitcoin::util::{psbt, taproot};
-use bitcoin::{secp256k1, PublicKey, XOnlyPublicKey};
+use bitcoin::bip32::{ChildNumber, DerivationPath, ExtendedPubKey, Fingerprint, KeySource};
+use bitcoin::{key::XOnlyPublicKey, secp256k1, PublicKey};
+use bitcoin::{psbt, taproot};
 use bitcoin::{Network, TxOut};
 
 use miniscript::descriptor::{
-    DefiniteDescriptorKey, DescriptorSecretKey, DescriptorType, InnerXKey, SinglePubKey,
+    DefiniteDescriptorKey, DescriptorMultiXKey, DescriptorSecretKey, DescriptorType,
+    DescriptorXKey, InnerXKey, KeyMap, SinglePubKey, Wildcard,
 };
 pub use miniscript::{
-    descriptor::DescriptorXKey, descriptor::KeyMap, descriptor::Wildcard, Descriptor,
-    DescriptorPublicKey, Legacy, Miniscript, ScriptContext, Segwitv0,
+    Descriptor, DescriptorPublicKey, Legacy, Miniscript, ScriptContext, Segwitv0,
 };
 use miniscript::{ForEachKey, MiniscriptKey, TranslatePk};
 
@@ -59,16 +59,16 @@ pub type DerivedDescriptor = Descriptor<DefiniteDescriptorKey>;
 /// Alias for the type of maps that represent derivation paths in a [`psbt::Input`] or
 /// [`psbt::Output`]
 ///
-/// [`psbt::Input`]: bitcoin::util::psbt::Input
-/// [`psbt::Output`]: bitcoin::util::psbt::Output
+/// [`psbt::Input`]: bitcoin::psbt::Input
+/// [`psbt::Output`]: bitcoin::psbt::Output
 pub type HdKeyPaths = BTreeMap<secp256k1::PublicKey, KeySource>;
 
 /// Alias for the type of maps that represent taproot key origins in a [`psbt::Input`] or
 /// [`psbt::Output`]
 ///
-/// [`psbt::Input`]: bitcoin::util::psbt::Input
-/// [`psbt::Output`]: bitcoin::util::psbt::Output
-pub type TapKeyOrigins = BTreeMap<bitcoin::XOnlyPublicKey, (Vec<taproot::TapLeafHash>, KeySource)>;
+/// [`psbt::Input`]: bitcoin::psbt::Input
+/// [`psbt::Output`]: bitcoin::psbt::Output
+pub type TapKeyOrigins = BTreeMap<XOnlyPublicKey, (Vec<taproot::TapLeafHash>, KeySource)>;
 
 /// Trait for types which can be converted into an [`ExtendedDescriptor`] and a [`KeyMap`] usable by a wallet in a specific [`Network`]
 pub trait IntoWalletDescriptor {
@@ -136,14 +136,10 @@ impl IntoWalletDescriptor for (ExtendedDescriptor, KeyMap) {
             network: Network,
         }
 
-        impl<'s, 'd>
-            miniscript::Translator<DescriptorPublicKey, miniscript::DummyKey, DescriptorError>
+        impl<'s, 'd> miniscript::Translator<DescriptorPublicKey, String, DescriptorError>
             for Translator<'s, 'd>
         {
-            fn pk(
-                &mut self,
-                pk: &DescriptorPublicKey,
-            ) -> Result<miniscript::DummyKey, DescriptorError> {
+            fn pk(&mut self, pk: &DescriptorPublicKey) -> Result<String, DescriptorError> {
                 let secp = &self.secp;
 
                 let (_, _, networks) = if self.descriptor.is_taproot() {
@@ -161,7 +157,7 @@ impl IntoWalletDescriptor for (ExtendedDescriptor, KeyMap) {
                 };
 
                 if networks.contains(&self.network) {
-                    Ok(miniscript::DummyKey)
+                    Ok(Default::default())
                 } else {
                     Err(DescriptorError::Key(KeyError::InvalidNetwork))
                 }
@@ -169,35 +165,40 @@ impl IntoWalletDescriptor for (ExtendedDescriptor, KeyMap) {
             fn sha256(
                 &mut self,
                 _sha256: &<DescriptorPublicKey as MiniscriptKey>::Sha256,
-            ) -> Result<miniscript::DummySha256Hash, DescriptorError> {
+            ) -> Result<String, DescriptorError> {
                 Ok(Default::default())
             }
             fn hash256(
                 &mut self,
                 _hash256: &<DescriptorPublicKey as MiniscriptKey>::Hash256,
-            ) -> Result<miniscript::DummyHash256Hash, DescriptorError> {
+            ) -> Result<String, DescriptorError> {
                 Ok(Default::default())
             }
             fn ripemd160(
                 &mut self,
                 _ripemd160: &<DescriptorPublicKey as MiniscriptKey>::Ripemd160,
-            ) -> Result<miniscript::DummyRipemd160Hash, DescriptorError> {
+            ) -> Result<String, DescriptorError> {
                 Ok(Default::default())
             }
             fn hash160(
                 &mut self,
                 _hash160: &<DescriptorPublicKey as MiniscriptKey>::Hash160,
-            ) -> Result<miniscript::DummyHash160Hash, DescriptorError> {
+            ) -> Result<String, DescriptorError> {
                 Ok(Default::default())
             }
         }
 
         // check the network for the keys
-        self.0.translate_pk(&mut Translator {
+        use miniscript::TranslateErr;
+        match self.0.translate_pk(&mut Translator {
             secp,
             network,
             descriptor: &self.0,
-        })?;
+        }) {
+            Ok(_) => {}
+            Err(TranslateErr::TranslatorErr(e)) => return Err(e),
+            Err(TranslateErr::OuterError(e)) => return Err(e.into()),
+        }
 
         Ok(self)
     }
@@ -251,7 +252,12 @@ impl IntoWalletDescriptor for DescriptorTemplateOut {
         }
 
         // fixup the network for keys that need it in the descriptor
-        let translated = desc.translate_pk(&mut Translator { network })?;
+        use miniscript::TranslateErr;
+        let translated = match desc.translate_pk(&mut Translator { network }) {
+            Ok(descriptor) => descriptor,
+            Err(TranslateErr::TranslatorErr(e)) => return Err(e),
+            Err(TranslateErr::OuterError(e)) => return Err(e.into()),
+        };
         // ...and in the key map
         let fixed_keymap = keymap
             .into_iter()
@@ -302,6 +308,10 @@ pub(crate) fn into_wallet_descriptor_checked<T: IntoWalletDescriptor>(
         return Err(DescriptorError::HardenedDerivationXpub);
     }
 
+    if descriptor.is_multipath() {
+        return Err(DescriptorError::MultiPath);
+    }
+
     // Run miniscript's sanity check, which will look for duplicated keys and other potential
     // issues
     descriptor.sanity_check()?;
@@ -340,6 +350,18 @@ pub(crate) trait XKeyUtils {
     fn root_fingerprint(&self, secp: &SecpCtx) -> Fingerprint;
 }
 
+impl<T> XKeyUtils for DescriptorMultiXKey<T>
+where
+    T: InnerXKey,
+{
+    fn root_fingerprint(&self, secp: &SecpCtx) -> Fingerprint {
+        match self.origin {
+            Some((fingerprint, _)) => fingerprint,
+            None => self.xkey.xkey_fingerprint(secp),
+        }
+    }
+}
+
 impl<T> XKeyUtils for DescriptorXKey<T>
 where
     T: InnerXKey,
@@ -466,11 +488,6 @@ impl DescriptorMeta for ExtendedDescriptor {
                         ) {
                             Some(derive_path)
                         } else {
-                            log::debug!(
-                                "Key `{}` derived with {} yields an unexpected key",
-                                root_fingerprint,
-                                derive_path
-                            );
                             None
                         }
                     });
@@ -494,7 +511,10 @@ impl DescriptorMeta for ExtendedDescriptor {
             false
         });
 
-        path_found.map(|path| self.at_derivation_index(path))
+        path_found.map(|path| {
+            self.at_derivation_index(path)
+                .expect("We ignore hardened wildcards")
+        })
     }
 
     fn derive_from_hd_keypaths(
@@ -545,7 +565,7 @@ impl DescriptorMeta for ExtendedDescriptor {
             return None;
         }
 
-        let descriptor = self.at_derivation_index(0);
+        let descriptor = self.at_derivation_index(0).expect("0 is not hardened");
         match descriptor.desc_type() {
             // TODO: add pk() here
             DescriptorType::Pkh
@@ -585,11 +605,10 @@ mod test {
     use core::str::FromStr;
 
     use assert_matches::assert_matches;
-    use bitcoin::consensus::encode::deserialize;
     use bitcoin::hashes::hex::FromHex;
     use bitcoin::secp256k1::Secp256k1;
-    use bitcoin::util::{bip32, psbt};
-    use bitcoin::Script;
+    use bitcoin::ScriptBuf;
+    use bitcoin::{bip32, psbt::Psbt};
 
     use super::*;
     use crate::psbt::PsbtUtils;
@@ -600,7 +619,7 @@ mod test {
             "wpkh(02b4632d08485ff1df2db55b9dafd23347d1c47a457072a1e87be26896549a8737)",
         )
         .unwrap();
-        let psbt: psbt::PartiallySignedTransaction = deserialize(
+        let psbt = Psbt::deserialize(
             &Vec::<u8>::from_hex(
                 "70736274ff010052010000000162307be8e431fbaff807cdf9cdc3fde44d7402\
                  11bc8342c31ffd6ec11fe35bcc0100000000ffffffff01328601000000000016\
@@ -623,7 +642,7 @@ mod test {
             "pkh([0f056943/44h/0h/0h]tpubDDpWvmUrPZrhSPmUzCMBHffvC3HyMAPnWDSAQNBTnj1iZeJa7BZQEttFiP4DS4GCcXQHezdXhn86Hj6LHX5EDstXPWrMaSneRWM8yUf6NFd/10/*)",
         )
         .unwrap();
-        let psbt: psbt::PartiallySignedTransaction = deserialize(
+        let psbt = Psbt::deserialize(
             &Vec::<u8>::from_hex(
                 "70736274ff010053010000000145843b86be54a3cd8c9e38444e1162676c00df\
                  e7964122a70df491ea12fd67090100000000ffffffff01c19598000000000017\
@@ -654,7 +673,7 @@ mod test {
             "wsh(and_v(v:pk(03b6633fef2397a0a9de9d7b6f23aef8368a6e362b0581f0f0af70d5ecfd254b14),older(6)))",
         )
         .unwrap();
-        let psbt: psbt::PartiallySignedTransaction = deserialize(
+        let psbt = Psbt::deserialize(
             &Vec::<u8>::from_hex(
                 "70736274ff01005302000000011c8116eea34408ab6529223c9a176606742207\
                  67a1ff1d46a6e3c4a88243ea6e01000000000600000001109698000000000017\
@@ -678,7 +697,7 @@ mod test {
             "sh(and_v(v:pk(021403881a5587297818fcaf17d239cefca22fce84a45b3b1d23e836c4af671dbb),after(630000)))",
         )
         .unwrap();
-        let psbt: psbt::PartiallySignedTransaction = deserialize(
+        let psbt = Psbt::deserialize(
             &Vec::<u8>::from_hex(
                 "70736274ff0100530100000001bc8c13df445dfadcc42afa6dc841f85d22b01d\
                  a6270ebf981740f4b7b1d800390000000000feffffff01ba9598000000000017\
@@ -845,6 +864,12 @@ mod test {
 
         assert_matches!(result, Err(DescriptorError::HardenedDerivationXpub));
 
+        let descriptor = "wpkh(tpubD6NzVbkrYhZ4XHndKkuB8FifXm8r5FQHwrN6oZuWCz13qb93rtgKvD4PQsqC4HP4yhV3tA2fqr2RbY5mNXfM7RxXUoeABoDtsFUq2zJq6YK/<0;1>/*)";
+        let result = into_wallet_descriptor_checked(descriptor, &secp, Network::Testnet);
+
+        assert_matches!(result, Err(DescriptorError::MultiPath));
+
+        // repeated pubkeys
         let descriptor = "wsh(multi(2,tpubD6NzVbkrYhZ4XHndKkuB8FifXm8r5FQHwrN6oZuWCz13qb93rtgKvD4PQsqC4HP4yhV3tA2fqr2RbY5mNXfM7RxXUoeABoDtsFUq2zJq6YK/0/*,tpubD6NzVbkrYhZ4XHndKkuB8FifXm8r5FQHwrN6oZuWCz13qb93rtgKvD4PQsqC4HP4yhV3tA2fqr2RbY5mNXfM7RxXUoeABoDtsFUq2zJq6YK/0/*))";
         let result = into_wallet_descriptor_checked(descriptor, &secp, Network::Testnet);
 
@@ -861,9 +886,9 @@ mod test {
         let (descriptor, _) =
             into_wallet_descriptor_checked(descriptor, &secp, Network::Testnet).unwrap();
 
-        let descriptor = descriptor.at_derivation_index(0);
+        let descriptor = descriptor.at_derivation_index(0).unwrap();
 
-        let script = Script::from_str("5321022f533b667e2ea3b36e21961c9fe9dca340fbe0af5210173a83ae0337ab20a57621026bb53a98e810bd0ee61a0ed1164ba6c024786d76554e793e202dc6ce9c78c4ea2102d5b8a7d66a41ffdb6f4c53d61994022e886b4f45001fb158b95c9164d45f8ca3210324b75eead2c1f9c60e8adeb5e7009fec7a29afcdb30d829d82d09562fe8bae8521032d34f8932200833487bd294aa219dcbe000b9f9b3d824799541430009f0fa55121037468f8ea99b6c64788398b5ad25480cad08f4b0d65be54ce3a55fd206b5ae4722103f72d3d96663b0ea99b0aeb0d7f273cab11a8de37885f1dddc8d9112adb87169357ae").unwrap();
+        let script = ScriptBuf::from_hex("5321022f533b667e2ea3b36e21961c9fe9dca340fbe0af5210173a83ae0337ab20a57621026bb53a98e810bd0ee61a0ed1164ba6c024786d76554e793e202dc6ce9c78c4ea2102d5b8a7d66a41ffdb6f4c53d61994022e886b4f45001fb158b95c9164d45f8ca3210324b75eead2c1f9c60e8adeb5e7009fec7a29afcdb30d829d82d09562fe8bae8521032d34f8932200833487bd294aa219dcbe000b9f9b3d824799541430009f0fa55121037468f8ea99b6c64788398b5ad25480cad08f4b0d65be54ce3a55fd206b5ae4722103f72d3d96663b0ea99b0aeb0d7f273cab11a8de37885f1dddc8d9112adb87169357ae").unwrap();
 
         let mut psbt_input = psbt::Input::default();
         psbt_input

crates/bdk/src/descriptor/policy.rs

@@ -33,21 +33,22 @@
 //! let signers = Arc::new(SignersContainer::build(key_map, &extended_desc, &secp));
 //! let policy = extended_desc.extract_policy(&signers, BuildSatisfaction::None, &secp)?;
 //! println!("policy: {}", serde_json::to_string(&policy).unwrap());
-//! # Ok::<(), bdk::Error>(())
+//! # Ok::<(), anyhow::Error>(())
 //! ```
 
 use crate::collections::{BTreeMap, HashSet, VecDeque};
 use alloc::string::String;
 use alloc::vec::Vec;
 use core::cmp::max;
+
 use core::fmt;
 
 use serde::ser::SerializeMap;
 use serde::{Serialize, Serializer};
 
+use bitcoin::bip32::Fingerprint;
 use bitcoin::hashes::{hash160, ripemd160, sha256};
-use bitcoin::util::bip32::Fingerprint;
-use bitcoin::{LockTime, PublicKey, Sequence, XOnlyPublicKey};
+use bitcoin::{absolute, key::XOnlyPublicKey, PublicKey, Sequence};
 
 use miniscript::descriptor::{
     DescriptorPublicKey, ShInner, SinglePub, SinglePubKey, SortedMultiVec, WshInner,
@@ -57,9 +58,6 @@ use miniscript::{
     Descriptor, Miniscript, Satisfier, ScriptContext, SigType, Terminal, ToPublicKey,
 };
 
-#[allow(unused_imports)]
-use log::{debug, error, info, trace};
-
 use crate::descriptor::ExtractPolicy;
 use crate::keys::ExtScriptContext;
 use crate::wallet::signer::{SignerId, SignersContainer};
@@ -68,7 +66,7 @@ use crate::wallet::utils::{After, Older, SecpCtx};
 use super::checksum::calc_checksum;
 use super::error::Error;
 use super::XKeyUtils;
-use bitcoin::util::psbt::{Input as PsbtInput, PartiallySignedTransaction as Psbt};
+use bitcoin::psbt::{self, Psbt};
 use miniscript::psbt::PsbtInputSatisfier;
 
 /// A unique identifier for a key
@@ -95,6 +93,9 @@ impl PkOrF {
                 ..
             }) => PkOrF::XOnlyPubkey(*pk),
             DescriptorPublicKey::XPub(xpub) => PkOrF::Fingerprint(xpub.root_fingerprint(secp)),
+            DescriptorPublicKey::MultiXPub(multi) => {
+                PkOrF::Fingerprint(multi.root_fingerprint(secp))
+            }
         }
     }
 }
@@ -131,7 +132,7 @@ pub enum SatisfiableItem {
     /// Absolute timeclock timestamp
     AbsoluteTimelock {
         /// The timelock value
-        value: LockTime,
+        value: absolute::LockTime,
     },
     /// Relative timelock locktime
     RelativeTimelock {
@@ -451,11 +452,14 @@ pub struct Condition {
     pub csv: Option<Sequence>,
     /// Optional timelock condition
     #[serde(skip_serializing_if = "Option::is_none")]
-    pub timelock: Option<LockTime>,
+    pub timelock: Option<absolute::LockTime>,
 }
 
 impl Condition {
-    fn merge_nlocktime(a: LockTime, b: LockTime) -> Result<LockTime, PolicyError> {
+    fn merge_nlocktime(
+        a: absolute::LockTime,
+        b: absolute::LockTime,
+    ) -> Result<absolute::LockTime, PolicyError> {
         if !a.is_same_unit(b) {
             Err(PolicyError::MixedTimelockUnits)
         } else if a > b {
@@ -515,7 +519,7 @@ pub enum PolicyError {
 impl fmt::Display for PolicyError {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         match self {
-            Self::NotEnoughItemsSelected(err) => write!(f, "Not enought items selected: {}", err),
+            Self::NotEnoughItemsSelected(err) => write!(f, "Not enough items selected: {}", err),
             Self::IndexOutOfRange(index) => write!(f, "Index out of range: {}", index),
             Self::AddOnLeaf => write!(f, "Add on leaf"),
             Self::AddOnPartialComplete => write!(f, "Add on partial complete"),
@@ -749,6 +753,7 @@ fn signer_id(key: &DescriptorPublicKey, secp: &SecpCtx) -> SignerId {
             ..
         }) => pk.to_pubkeyhash(SigType::Ecdsa).into(),
         DescriptorPublicKey::XPub(xpub) => xpub.root_fingerprint(secp).into(),
+        DescriptorPublicKey::MultiXPub(xpub) => xpub.root_fingerprint(secp).into(),
     }
 }
 
@@ -786,9 +791,9 @@ fn make_generic_signature<M: Fn() -> SatisfiableItem, F: Fn(&Psbt) -> bool>(
 fn generic_sig_in_psbt<
     // C is for "check", it's a closure we use to *check* if a psbt input contains the signature
     // for a specific key
-    C: Fn(&PsbtInput, &SinglePubKey) -> bool,
+    C: Fn(&psbt::Input, &SinglePubKey) -> bool,
     // E is for "extract", it extracts a key from the bip32 derivations found in the psbt input
-    E: Fn(&PsbtInput, Fingerprint) -> Option<SinglePubKey>,
+    E: Fn(&psbt::Input, Fingerprint) -> Option<SinglePubKey>,
 >(
     psbt: &Psbt,
     key: &DescriptorPublicKey,
@@ -806,6 +811,13 @@ fn generic_sig_in_psbt<
                 None => false,
             }
         }
+        DescriptorPublicKey::MultiXPub(xpub) => {
+            //TODO check actual derivation matches
+            match extract(input, xpub.root_fingerprint(secp)) {
+                Some(pubkey) => check(input, &pubkey),
+                None => false,
+            }
+        }
     })
 }
 
@@ -911,12 +923,12 @@ impl<Ctx: ScriptContext + 'static> ExtractPolicy for Miniscript<DescriptorPublic
             }
             Terminal::After(value) => {
                 let mut policy: Policy = SatisfiableItem::AbsoluteTimelock {
-                    value: value.into(),
+                    value: (*value).into(),
                 }
                 .into();
                 policy.contribution = Satisfaction::Complete {
                     condition: Condition {
-                        timelock: Some(value.into()),
+                        timelock: Some((*value).into()),
                         csv: None,
                     },
                 };
@@ -928,9 +940,9 @@ impl<Ctx: ScriptContext + 'static> ExtractPolicy for Miniscript<DescriptorPublic
                 {
                     let after = After::new(Some(current_height), false);
                     let after_sat =
-                        Satisfier::<bitcoin::PublicKey>::check_after(&after, value.into());
+                        Satisfier::<bitcoin::PublicKey>::check_after(&after, (*value).into());
                     let inputs_sat = psbt_inputs_sat(psbt).all(|sat| {
-                        Satisfier::<bitcoin::PublicKey>::check_after(&sat, value.into())
+                        Satisfier::<bitcoin::PublicKey>::check_after(&sat, (*value).into())
                     });
                     if after_sat && inputs_sat {
                         policy.satisfaction = policy.contribution.clone();
@@ -1156,8 +1168,8 @@ mod test {
     use crate::wallet::signer::SignersContainer;
     use alloc::{string::ToString, sync::Arc};
     use assert_matches::assert_matches;
+    use bitcoin::bip32;
     use bitcoin::secp256k1::Secp256k1;
-    use bitcoin::util::bip32;
     use bitcoin::Network;
     use core::str::FromStr;
 
@@ -1575,6 +1587,7 @@ mod test {
 
         let addr = wallet_desc
             .at_derivation_index(0)
+            .unwrap()
             .address(Network::Testnet)
             .unwrap();
         assert_eq!(
@@ -1641,6 +1654,7 @@ mod test {
 
         let addr = wallet_desc
             .at_derivation_index(0)
+            .unwrap()
             .address(Network::Testnet)
             .unwrap();
         assert_eq!(

crates/bdk/src/descriptor/template.rs

@@ -14,7 +14,7 @@
 //! This module contains the definition of various common script templates that are ready to be
 //! used. See the documentation of each template for an example.
 
-use bitcoin::util::bip32;
+use bitcoin::bip32;
 use bitcoin::Network;
 
 use miniscript::{Legacy, Segwitv0, Tap};
@@ -195,7 +195,7 @@ impl<K: IntoDescriptorKey<Tap>> DescriptorTemplate for P2TR<K> {
 /// # use bdk::wallet::AddressIndex::New;
 /// use bdk::template::Bip44;
 ///
-/// let key = bitcoin::util::bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPeZRHk4rTG6orPS2CRNFX3njhUXx5vj9qGog5ZMH4uGReDWN5kCkY3jmWEtWause41CDvBRXD1shKknAMKxT99o9qUTRVC6m")?;
+/// let key = bitcoin::bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPeZRHk4rTG6orPS2CRNFX3njhUXx5vj9qGog5ZMH4uGReDWN5kCkY3jmWEtWause41CDvBRXD1shKknAMKxT99o9qUTRVC6m")?;
 /// let mut wallet = Wallet::new_no_persist(
 ///     Bip44(key.clone(), KeychainKind::External),
 ///     Some(Bip44(key, KeychainKind::Internal)),
@@ -232,8 +232,8 @@ impl<K: DerivableKey<Legacy>> DescriptorTemplate for Bip44<K> {
 /// # use bdk::wallet::AddressIndex::New;
 /// use bdk::template::Bip44Public;
 ///
-/// let key = bitcoin::util::bip32::ExtendedPubKey::from_str("tpubDDDzQ31JkZB7VxUr9bjvBivDdqoFLrDPyLWtLapArAi51ftfmCb2DPxwLQzX65iNcXz1DGaVvyvo6JQ6rTU73r2gqdEo8uov9QKRb7nKCSU")?;
-/// let fingerprint = bitcoin::util::bip32::Fingerprint::from_str("c55b303f")?;
+/// let key = bitcoin::bip32::ExtendedPubKey::from_str("tpubDDDzQ31JkZB7VxUr9bjvBivDdqoFLrDPyLWtLapArAi51ftfmCb2DPxwLQzX65iNcXz1DGaVvyvo6JQ6rTU73r2gqdEo8uov9QKRb7nKCSU")?;
+/// let fingerprint = bitcoin::bip32::Fingerprint::from_str("c55b303f")?;
 /// let mut wallet = Wallet::new_no_persist(
 ///     Bip44Public(key.clone(), fingerprint, KeychainKind::External),
 ///     Some(Bip44Public(key, fingerprint, KeychainKind::Internal)),
@@ -270,7 +270,7 @@ impl<K: DerivableKey<Legacy>> DescriptorTemplate for Bip44Public<K> {
 /// # use bdk::wallet::AddressIndex::New;
 /// use bdk::template::Bip49;
 ///
-/// let key = bitcoin::util::bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPeZRHk4rTG6orPS2CRNFX3njhUXx5vj9qGog5ZMH4uGReDWN5kCkY3jmWEtWause41CDvBRXD1shKknAMKxT99o9qUTRVC6m")?;
+/// let key = bitcoin::bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPeZRHk4rTG6orPS2CRNFX3njhUXx5vj9qGog5ZMH4uGReDWN5kCkY3jmWEtWause41CDvBRXD1shKknAMKxT99o9qUTRVC6m")?;
 /// let mut wallet = Wallet::new_no_persist(
 ///     Bip49(key.clone(), KeychainKind::External),
 ///     Some(Bip49(key, KeychainKind::Internal)),
@@ -307,8 +307,8 @@ impl<K: DerivableKey<Segwitv0>> DescriptorTemplate for Bip49<K> {
 /// # use bdk::wallet::AddressIndex::New;
 /// use bdk::template::Bip49Public;
 ///
-/// let key = bitcoin::util::bip32::ExtendedPubKey::from_str("tpubDC49r947KGK52X5rBWS4BLs5m9SRY3pYHnvRrm7HcybZ3BfdEsGFyzCMzayi1u58eT82ZeyFZwH7DD6Q83E3fM9CpfMtmnTygnLfP59jL9L")?;
-/// let fingerprint = bitcoin::util::bip32::Fingerprint::from_str("c55b303f")?;
+/// let key = bitcoin::bip32::ExtendedPubKey::from_str("tpubDC49r947KGK52X5rBWS4BLs5m9SRY3pYHnvRrm7HcybZ3BfdEsGFyzCMzayi1u58eT82ZeyFZwH7DD6Q83E3fM9CpfMtmnTygnLfP59jL9L")?;
+/// let fingerprint = bitcoin::bip32::Fingerprint::from_str("c55b303f")?;
 /// let mut wallet = Wallet::new_no_persist(
 ///     Bip49Public(key.clone(), fingerprint, KeychainKind::External),
 ///     Some(Bip49Public(key, fingerprint, KeychainKind::Internal)),
@@ -345,7 +345,7 @@ impl<K: DerivableKey<Segwitv0>> DescriptorTemplate for Bip49Public<K> {
 /// # use bdk::wallet::AddressIndex::New;
 /// use bdk::template::Bip84;
 ///
-/// let key = bitcoin::util::bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPeZRHk4rTG6orPS2CRNFX3njhUXx5vj9qGog5ZMH4uGReDWN5kCkY3jmWEtWause41CDvBRXD1shKknAMKxT99o9qUTRVC6m")?;
+/// let key = bitcoin::bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPeZRHk4rTG6orPS2CRNFX3njhUXx5vj9qGog5ZMH4uGReDWN5kCkY3jmWEtWause41CDvBRXD1shKknAMKxT99o9qUTRVC6m")?;
 /// let mut wallet = Wallet::new_no_persist(
 ///     Bip84(key.clone(), KeychainKind::External),
 ///     Some(Bip84(key, KeychainKind::Internal)),
@@ -382,8 +382,8 @@ impl<K: DerivableKey<Segwitv0>> DescriptorTemplate for Bip84<K> {
 /// # use bdk::wallet::AddressIndex::New;
 /// use bdk::template::Bip84Public;
 ///
-/// let key = bitcoin::util::bip32::ExtendedPubKey::from_str("tpubDC2Qwo2TFsaNC4ju8nrUJ9mqVT3eSgdmy1yPqhgkjwmke3PRXutNGRYAUo6RCHTcVQaDR3ohNU9we59brGHuEKPvH1ags2nevW5opEE9Z5Q")?;
-/// let fingerprint = bitcoin::util::bip32::Fingerprint::from_str("c55b303f")?;
+/// let key = bitcoin::bip32::ExtendedPubKey::from_str("tpubDC2Qwo2TFsaNC4ju8nrUJ9mqVT3eSgdmy1yPqhgkjwmke3PRXutNGRYAUo6RCHTcVQaDR3ohNU9we59brGHuEKPvH1ags2nevW5opEE9Z5Q")?;
+/// let fingerprint = bitcoin::bip32::Fingerprint::from_str("c55b303f")?;
 /// let mut wallet = Wallet::new_no_persist(
 ///     Bip84Public(key.clone(), fingerprint, KeychainKind::External),
 ///     Some(Bip84Public(key, fingerprint, KeychainKind::Internal)),
@@ -420,7 +420,7 @@ impl<K: DerivableKey<Segwitv0>> DescriptorTemplate for Bip84Public<K> {
 /// # use bdk::wallet::AddressIndex::New;
 /// use bdk::template::Bip86;
 ///
-/// let key = bitcoin::util::bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPeZRHk4rTG6orPS2CRNFX3njhUXx5vj9qGog5ZMH4uGReDWN5kCkY3jmWEtWause41CDvBRXD1shKknAMKxT99o9qUTRVC6m")?;
+/// let key = bitcoin::bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPeZRHk4rTG6orPS2CRNFX3njhUXx5vj9qGog5ZMH4uGReDWN5kCkY3jmWEtWause41CDvBRXD1shKknAMKxT99o9qUTRVC6m")?;
 /// let mut wallet = Wallet::new_no_persist(
 ///     Bip86(key.clone(), KeychainKind::External),
 ///     Some(Bip86(key, KeychainKind::Internal)),
@@ -457,8 +457,8 @@ impl<K: DerivableKey<Tap>> DescriptorTemplate for Bip86<K> {
 /// # use bdk::wallet::AddressIndex::New;
 /// use bdk::template::Bip86Public;
 ///
-/// let key = bitcoin::util::bip32::ExtendedPubKey::from_str("tpubDC2Qwo2TFsaNC4ju8nrUJ9mqVT3eSgdmy1yPqhgkjwmke3PRXutNGRYAUo6RCHTcVQaDR3ohNU9we59brGHuEKPvH1ags2nevW5opEE9Z5Q")?;
-/// let fingerprint = bitcoin::util::bip32::Fingerprint::from_str("c55b303f")?;
+/// let key = bitcoin::bip32::ExtendedPubKey::from_str("tpubDC2Qwo2TFsaNC4ju8nrUJ9mqVT3eSgdmy1yPqhgkjwmke3PRXutNGRYAUo6RCHTcVQaDR3ohNU9we59brGHuEKPvH1ags2nevW5opEE9Z5Q")?;
+/// let fingerprint = bitcoin::bip32::Fingerprint::from_str("c55b303f")?;
 /// let mut wallet = Wallet::new_no_persist(
 ///     Bip86Public(key.clone(), fingerprint, KeychainKind::External),
 ///     Some(Bip86Public(key, fingerprint, KeychainKind::Internal)),
@@ -565,31 +565,31 @@ mod test {
     // BIP44 `pkh(key/44'/{0,1}'/0'/{0,1}/*)`
     #[test]
     fn test_bip44_template_cointype() {
-        use bitcoin::util::bip32::ChildNumber::{self, Hardened};
+        use bitcoin::bip32::ChildNumber::{self, Hardened};
 
-        let xprvkey = bitcoin::util::bip32::ExtendedPrivKey::from_str("xprv9s21ZrQH143K2fpbqApQL69a4oKdGVnVN52R82Ft7d1pSqgKmajF62acJo3aMszZb6qQ22QsVECSFxvf9uyxFUvFYQMq3QbtwtRSMjLAhMf").unwrap();
+        let xprvkey = bitcoin::bip32::ExtendedPrivKey::from_str("xprv9s21ZrQH143K2fpbqApQL69a4oKdGVnVN52R82Ft7d1pSqgKmajF62acJo3aMszZb6qQ22QsVECSFxvf9uyxFUvFYQMq3QbtwtRSMjLAhMf").unwrap();
         assert_eq!(Network::Bitcoin, xprvkey.network);
         let xdesc = Bip44(xprvkey, KeychainKind::Internal)
             .build(Network::Bitcoin)
             .unwrap();
 
         if let ExtendedDescriptor::Pkh(pkh) = xdesc.0 {
-            let path: Vec<ChildNumber> = pkh.into_inner().full_derivation_path().into();
-            let purpose = path.get(0).unwrap();
+            let path: Vec<ChildNumber> = pkh.into_inner().full_derivation_path().unwrap().into();
+            let purpose = path.first().unwrap();
             assert_matches!(purpose, Hardened { index: 44 });
             let coin_type = path.get(1).unwrap();
             assert_matches!(coin_type, Hardened { index: 0 });
         }
 
-        let tprvkey = bitcoin::util::bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
+        let tprvkey = bitcoin::bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
         assert_eq!(Network::Testnet, tprvkey.network);
         let tdesc = Bip44(tprvkey, KeychainKind::Internal)
             .build(Network::Testnet)
             .unwrap();
 
         if let ExtendedDescriptor::Pkh(pkh) = tdesc.0 {
-            let path: Vec<ChildNumber> = pkh.into_inner().full_derivation_path().into();
-            let purpose = path.get(0).unwrap();
+            let path: Vec<ChildNumber> = pkh.into_inner().full_derivation_path().unwrap().into();
+            let purpose = path.first().unwrap();
             assert_matches!(purpose, Hardened { index: 44 });
             let coin_type = path.get(1).unwrap();
             assert_matches!(coin_type, Hardened { index: 1 });
@@ -612,9 +612,9 @@ mod test {
         for i in 0..expected.len() {
             let index = i as u32;
             let child_desc = if !desc.has_wildcard() {
-                desc.at_derivation_index(0)
+                desc.at_derivation_index(0).unwrap()
             } else {
-                desc.at_derivation_index(index)
+                desc.at_derivation_index(index).unwrap()
             };
             let address = child_desc.address(network).unwrap();
             assert_eq!(address.to_string(), *expected.get(i).unwrap());
@@ -740,7 +740,7 @@ mod test {
     // BIP44 `pkh(key/44'/0'/0'/{0,1}/*)`
     #[test]
     fn test_bip44_template() {
-        let prvkey = bitcoin::util::bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
+        let prvkey = bitcoin::bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
         check(
             Bip44(prvkey, KeychainKind::External).build(Network::Bitcoin),
             false,
@@ -770,8 +770,8 @@ mod test {
     // BIP44 public `pkh(key/{0,1}/*)`
     #[test]
     fn test_bip44_public_template() {
-        let pubkey = bitcoin::util::bip32::ExtendedPubKey::from_str("tpubDDDzQ31JkZB7VxUr9bjvBivDdqoFLrDPyLWtLapArAi51ftfmCb2DPxwLQzX65iNcXz1DGaVvyvo6JQ6rTU73r2gqdEo8uov9QKRb7nKCSU").unwrap();
-        let fingerprint = bitcoin::util::bip32::Fingerprint::from_str("c55b303f").unwrap();
+        let pubkey = bitcoin::bip32::ExtendedPubKey::from_str("tpubDDDzQ31JkZB7VxUr9bjvBivDdqoFLrDPyLWtLapArAi51ftfmCb2DPxwLQzX65iNcXz1DGaVvyvo6JQ6rTU73r2gqdEo8uov9QKRb7nKCSU").unwrap();
+        let fingerprint = bitcoin::bip32::Fingerprint::from_str("c55b303f").unwrap();
         check(
             Bip44Public(pubkey, fingerprint, KeychainKind::External).build(Network::Bitcoin),
             false,
@@ -801,7 +801,7 @@ mod test {
     // BIP49 `sh(wpkh(key/49'/0'/0'/{0,1}/*))`
     #[test]
     fn test_bip49_template() {
-        let prvkey = bitcoin::util::bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
+        let prvkey = bitcoin::bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
         check(
             Bip49(prvkey, KeychainKind::External).build(Network::Bitcoin),
             true,
@@ -831,8 +831,8 @@ mod test {
     // BIP49 public `sh(wpkh(key/{0,1}/*))`
     #[test]
     fn test_bip49_public_template() {
-        let pubkey = bitcoin::util::bip32::ExtendedPubKey::from_str("tpubDC49r947KGK52X5rBWS4BLs5m9SRY3pYHnvRrm7HcybZ3BfdEsGFyzCMzayi1u58eT82ZeyFZwH7DD6Q83E3fM9CpfMtmnTygnLfP59jL9L").unwrap();
-        let fingerprint = bitcoin::util::bip32::Fingerprint::from_str("c55b303f").unwrap();
+        let pubkey = bitcoin::bip32::ExtendedPubKey::from_str("tpubDC49r947KGK52X5rBWS4BLs5m9SRY3pYHnvRrm7HcybZ3BfdEsGFyzCMzayi1u58eT82ZeyFZwH7DD6Q83E3fM9CpfMtmnTygnLfP59jL9L").unwrap();
+        let fingerprint = bitcoin::bip32::Fingerprint::from_str("c55b303f").unwrap();
         check(
             Bip49Public(pubkey, fingerprint, KeychainKind::External).build(Network::Bitcoin),
             true,
@@ -862,7 +862,7 @@ mod test {
     // BIP84 `wpkh(key/84'/0'/0'/{0,1}/*)`
     #[test]
     fn test_bip84_template() {
-        let prvkey = bitcoin::util::bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
+        let prvkey = bitcoin::bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
         check(
             Bip84(prvkey, KeychainKind::External).build(Network::Bitcoin),
             true,
@@ -892,8 +892,8 @@ mod test {
     // BIP84 public `wpkh(key/{0,1}/*)`
     #[test]
     fn test_bip84_public_template() {
-        let pubkey = bitcoin::util::bip32::ExtendedPubKey::from_str("tpubDC2Qwo2TFsaNC4ju8nrUJ9mqVT3eSgdmy1yPqhgkjwmke3PRXutNGRYAUo6RCHTcVQaDR3ohNU9we59brGHuEKPvH1ags2nevW5opEE9Z5Q").unwrap();
-        let fingerprint = bitcoin::util::bip32::Fingerprint::from_str("c55b303f").unwrap();
+        let pubkey = bitcoin::bip32::ExtendedPubKey::from_str("tpubDC2Qwo2TFsaNC4ju8nrUJ9mqVT3eSgdmy1yPqhgkjwmke3PRXutNGRYAUo6RCHTcVQaDR3ohNU9we59brGHuEKPvH1ags2nevW5opEE9Z5Q").unwrap();
+        let fingerprint = bitcoin::bip32::Fingerprint::from_str("c55b303f").unwrap();
         check(
             Bip84Public(pubkey, fingerprint, KeychainKind::External).build(Network::Bitcoin),
             true,
@@ -924,7 +924,7 @@ mod test {
     // Used addresses in test vector in https://github.com/bitcoin/bips/blob/master/bip-0086.mediawiki
     #[test]
     fn test_bip86_template() {
-        let prvkey = bitcoin::util::bip32::ExtendedPrivKey::from_str("xprv9s21ZrQH143K3GJpoapnV8SFfukcVBSfeCficPSGfubmSFDxo1kuHnLisriDvSnRRuL2Qrg5ggqHKNVpxR86QEC8w35uxmGoggxtQTPvfUu").unwrap();
+        let prvkey = bitcoin::bip32::ExtendedPrivKey::from_str("xprv9s21ZrQH143K3GJpoapnV8SFfukcVBSfeCficPSGfubmSFDxo1kuHnLisriDvSnRRuL2Qrg5ggqHKNVpxR86QEC8w35uxmGoggxtQTPvfUu").unwrap();
         check(
             Bip86(prvkey, KeychainKind::External).build(Network::Bitcoin),
             false,
@@ -955,8 +955,8 @@ mod test {
     // Used addresses in test vector in https://github.com/bitcoin/bips/blob/master/bip-0086.mediawiki
     #[test]
     fn test_bip86_public_template() {
-        let pubkey = bitcoin::util::bip32::ExtendedPubKey::from_str("xpub6BgBgsespWvERF3LHQu6CnqdvfEvtMcQjYrcRzx53QJjSxarj2afYWcLteoGVky7D3UKDP9QyrLprQ3VCECoY49yfdDEHGCtMMj92pReUsQ").unwrap();
-        let fingerprint = bitcoin::util::bip32::Fingerprint::from_str("73c5da0a").unwrap();
+        let pubkey = bitcoin::bip32::ExtendedPubKey::from_str("xpub6BgBgsespWvERF3LHQu6CnqdvfEvtMcQjYrcRzx53QJjSxarj2afYWcLteoGVky7D3UKDP9QyrLprQ3VCECoY49yfdDEHGCtMMj92pReUsQ").unwrap();
+        let fingerprint = bitcoin::bip32::Fingerprint::from_str("73c5da0a").unwrap();
         check(
             Bip86Public(pubkey, fingerprint, KeychainKind::External).build(Network::Bitcoin),
             false,

crates/bdk/src/error.rs

@@ -1,201 +0,0 @@
-// Bitcoin Dev Kit
-// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
-//
-// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
-//
-// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
-// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
-// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
-// You may not use this file except in accordance with one or both of these
-// licenses.
-
-use crate::bitcoin::Network;
-use crate::{descriptor, wallet};
-use alloc::{string::String, vec::Vec};
-use bitcoin::{OutPoint, Txid};
-use core::fmt;
-
-/// Errors that can be thrown by the [`Wallet`](crate::wallet::Wallet)
-#[derive(Debug)]
-pub enum Error {
-    /// Generic error
-    Generic(String),
-    /// Cannot build a tx without recipients
-    NoRecipients,
-    /// `manually_selected_only` option is selected but no utxo has been passed
-    NoUtxosSelected,
-    /// Output created is under the dust limit, 546 satoshis
-    OutputBelowDustLimit(usize),
-    /// Wallet's UTXO set is not enough to cover recipient's requested plus fee
-    InsufficientFunds {
-        /// Sats needed for some transaction
-        needed: u64,
-        /// Sats available for spending
-        available: u64,
-    },
-    /// Branch and bound coin selection possible attempts with sufficiently big UTXO set could grow
-    /// exponentially, thus a limit is set, and when hit, this error is thrown
-    BnBTotalTriesExceeded,
-    /// Branch and bound coin selection tries to avoid needing a change by finding the right inputs for
-    /// the desired outputs plus fee, if there is not such combination this error is thrown
-    BnBNoExactMatch,
-    /// Happens when trying to spend an UTXO that is not in the internal database
-    UnknownUtxo,
-    /// Thrown when a tx is not found in the internal database
-    TransactionNotFound,
-    /// Happens when trying to bump a transaction that is already confirmed
-    TransactionConfirmed,
-    /// Trying to replace a tx that has a sequence >= `0xFFFFFFFE`
-    IrreplaceableTransaction,
-    /// When bumping a tx the fee rate requested is lower than required
-    FeeRateTooLow {
-        /// Required fee rate (satoshi/vbyte)
-        required: crate::types::FeeRate,
-    },
-    /// When bumping a tx the absolute fee requested is lower than replaced tx absolute fee
-    FeeTooLow {
-        /// Required fee absolute value (satoshi)
-        required: u64,
-    },
-    /// Node doesn't have data to estimate a fee rate
-    FeeRateUnavailable,
-    /// In order to use the [`TxBuilder::add_global_xpubs`] option every extended
-    /// key in the descriptor must either be a master key itself (having depth = 0) or have an
-    /// explicit origin provided
-    ///
-    /// [`TxBuilder::add_global_xpubs`]: crate::wallet::tx_builder::TxBuilder::add_global_xpubs
-    MissingKeyOrigin(String),
-    /// Error while working with [`keys`](crate::keys)
-    Key(crate::keys::KeyError),
-    /// Descriptor checksum mismatch
-    ChecksumMismatch,
-    /// Spending policy is not compatible with this [`KeychainKind`](crate::types::KeychainKind)
-    SpendingPolicyRequired(crate::types::KeychainKind),
-    /// Error while extracting and manipulating policies
-    InvalidPolicyPathError(crate::descriptor::policy::PolicyError),
-    /// Signing error
-    Signer(crate::wallet::signer::SignerError),
-    /// Requested outpoint doesn't exist in the tx (vout greater than available outputs)
-    InvalidOutpoint(OutPoint),
-    /// Error related to the parsing and usage of descriptors
-    Descriptor(crate::descriptor::error::Error),
-    /// Miniscript error
-    Miniscript(miniscript::Error),
-    /// Miniscript PSBT error
-    MiniscriptPsbt(MiniscriptPsbtError),
-    /// BIP32 error
-    Bip32(bitcoin::util::bip32::Error),
-    /// Partially signed bitcoin transaction error
-    Psbt(bitcoin::util::psbt::Error),
-}
-
-/// Errors returned by miniscript when updating inconsistent PSBTs
-#[derive(Debug, Clone)]
-pub enum MiniscriptPsbtError {
-    Conversion(miniscript::descriptor::ConversionError),
-    UtxoUpdate(miniscript::psbt::UtxoUpdateError),
-    OutputUpdate(miniscript::psbt::OutputUpdateError),
-}
-
-impl fmt::Display for MiniscriptPsbtError {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        match self {
-            Self::Conversion(err) => write!(f, "Conversion error: {}", err),
-            Self::UtxoUpdate(err) => write!(f, "UTXO update error: {}", err),
-            Self::OutputUpdate(err) => write!(f, "Output update error: {}", err),
-        }
-    }
-}
-
-#[cfg(feature = "std")]
-impl std::error::Error for MiniscriptPsbtError {}
-
-#[cfg(feature = "std")]
-impl fmt::Display for Error {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        match self {
-            Self::Generic(err) => write!(f, "Generic error: {}", err),
-            Self::NoRecipients => write!(f, "Cannot build tx without recipients"),
-            Self::NoUtxosSelected => write!(f, "No UTXO selected"),
-            Self::OutputBelowDustLimit(limit) => {
-                write!(f, "Output below the dust limit: {}", limit)
-            }
-            Self::InsufficientFunds { needed, available } => write!(
-                f,
-                "Insufficient funds: {} sat available of {} sat needed",
-                available, needed
-            ),
-            Self::BnBTotalTriesExceeded => {
-                write!(f, "Branch and bound coin selection: total tries exceeded")
-            }
-            Self::BnBNoExactMatch => write!(f, "Branch and bound coin selection: not exact match"),
-            Self::UnknownUtxo => write!(f, "UTXO not found in the internal database"),
-            Self::TransactionNotFound => {
-                write!(f, "Transaction not found in the internal database")
-            }
-            Self::TransactionConfirmed => write!(f, "Transaction already confirmed"),
-            Self::IrreplaceableTransaction => write!(f, "Transaction can't be replaced"),
-            Self::FeeRateTooLow { required } => write!(
-                f,
-                "Fee rate too low: required {} sat/vbyte",
-                required.as_sat_per_vb()
-            ),
-            Self::FeeTooLow { required } => write!(f, "Fee to low: required {} sat", required),
-            Self::FeeRateUnavailable => write!(f, "Fee rate unavailable"),
-            Self::MissingKeyOrigin(err) => write!(f, "Missing key origin: {}", err),
-            Self::Key(err) => write!(f, "Key error: {}", err),
-            Self::ChecksumMismatch => write!(f, "Descriptor checksum mismatch"),
-            Self::SpendingPolicyRequired(keychain_kind) => {
-                write!(f, "Spending policy required: {:?}", keychain_kind)
-            }
-            Self::InvalidPolicyPathError(err) => write!(f, "Invalid policy path: {}", err),
-            Self::Signer(err) => write!(f, "Signer error: {}", err),
-            Self::InvalidOutpoint(outpoint) => write!(
-                f,
-                "Requested outpoint doesn't exist in the tx: {}",
-                outpoint
-            ),
-            Self::Descriptor(err) => write!(f, "Descriptor error: {}", err),
-            Self::Miniscript(err) => write!(f, "Miniscript error: {}", err),
-            Self::MiniscriptPsbt(err) => write!(f, "Miniscript PSBT error: {}", err),
-            Self::Bip32(err) => write!(f, "BIP32 error: {}", err),
-            Self::Psbt(err) => write!(f, "PSBT error: {}", err),
-        }
-    }
-}
-
-#[cfg(feature = "std")]
-impl std::error::Error for Error {}
-
-macro_rules! impl_error {
-    ( $from:ty, $to:ident ) => {
-        impl_error!($from, $to, Error);
-    };
-    ( $from:ty, $to:ident, $impl_for:ty ) => {
-        impl core::convert::From<$from> for $impl_for {
-            fn from(err: $from) -> Self {
-                <$impl_for>::$to(err)
-            }
-        }
-    };
-}
-
-impl_error!(descriptor::error::Error, Descriptor);
-impl_error!(descriptor::policy::PolicyError, InvalidPolicyPathError);
-impl_error!(wallet::signer::SignerError, Signer);
-
-impl From<crate::keys::KeyError> for Error {
-    fn from(key_error: crate::keys::KeyError) -> Error {
-        match key_error {
-            crate::keys::KeyError::Miniscript(inner) => Error::Miniscript(inner),
-            crate::keys::KeyError::Bip32(inner) => Error::Bip32(inner),
-            crate::keys::KeyError::InvalidChecksum => Error::ChecksumMismatch,
-            e => Error::Key(e),
-        }
-    }
-}
-
-impl_error!(miniscript::Error, Miniscript);
-impl_error!(MiniscriptPsbtError, MiniscriptPsbt);
-impl_error!(bitcoin::util::bip32::Error, Bip32);
-impl_error!(bitcoin::util::psbt::Error, Psbt);

crates/bdk/src/keys/bip39.rs

@@ -15,7 +15,7 @@
 // something that should be fairly simple to re-implement.
 
 use alloc::string::String;
-use bitcoin::util::bip32;
+use bitcoin::bip32;
 use bitcoin::Network;
 
 use miniscript::ScriptContext;
@@ -142,7 +142,7 @@ impl<Ctx: ScriptContext> GeneratableKey<Ctx> for Mnemonic {
         (word_count, language): Self::Options,
         entropy: Self::Entropy,
     ) -> Result<GeneratedKey<Self, Ctx>, Self::Error> {
-        let entropy = &entropy.as_ref()[..(word_count as usize / 8)];
+        let entropy = &entropy[..(word_count as usize / 8)];
         let mnemonic = Mnemonic::from_entropy_in(language, entropy)?;
 
         Ok(GeneratedKey::new(mnemonic, any_network()))
@@ -154,7 +154,7 @@ mod test {
     use alloc::string::ToString;
     use core::str::FromStr;
 
-    use bitcoin::util::bip32;
+    use bitcoin::bip32;
 
     use bip39::{Language, Mnemonic};
 

crates/bdk/src/keys/mod.rs

@@ -22,8 +22,8 @@ use core::str::FromStr;
 
 use bitcoin::secp256k1::{self, Secp256k1, Signing};
 
-use bitcoin::util::bip32;
-use bitcoin::{Network, PrivateKey, PublicKey, XOnlyPublicKey};
+use bitcoin::bip32;
+use bitcoin::{key::XOnlyPublicKey, Network, PrivateKey, PublicKey};
 
 use miniscript::descriptor::{Descriptor, DescriptorXKey, Wildcard};
 pub use miniscript::descriptor::{
@@ -388,12 +388,12 @@ impl<Ctx: ScriptContext> From<bip32::ExtendedPrivKey> for ExtendedKey<Ctx> {
 ///
 /// ```
 /// use bdk::bitcoin;
-/// use bdk::bitcoin::util::bip32;
+/// use bdk::bitcoin::bip32;
 /// use bdk::keys::{DerivableKey, ExtendedKey, KeyError, ScriptContext};
 ///
 /// struct MyCustomKeyType {
 ///     key_data: bitcoin::PrivateKey,
-///     chain_code: Vec<u8>,
+///     chain_code: [u8; 32],
 ///     network: bitcoin::Network,
 /// }
 ///
@@ -404,7 +404,7 @@ impl<Ctx: ScriptContext> From<bip32::ExtendedPrivKey> for ExtendedKey<Ctx> {
 ///             depth: 0,
 ///             parent_fingerprint: bip32::Fingerprint::default(),
 ///             private_key: self.key_data.inner,
-///             chain_code: bip32::ChainCode::from(self.chain_code.as_ref()),
+///             chain_code: bip32::ChainCode::from(&self.chain_code),
 ///             child_number: bip32::ChildNumber::Normal { index: 0 },
 ///         };
 ///
@@ -413,20 +413,20 @@ impl<Ctx: ScriptContext> From<bip32::ExtendedPrivKey> for ExtendedKey<Ctx> {
 /// }
 /// ```
 ///
-/// Types that don't internally encode the [`Network`](bitcoin::Network) in which they are valid need some extra
+/// Types that don't internally encode the [`Network`] in which they are valid need some extra
 /// steps to override the set of valid networks, otherwise only the network specified in the
 /// [`ExtendedPrivKey`] or [`ExtendedPubKey`] will be considered valid.
 ///
 /// ```
 /// use bdk::bitcoin;
-/// use bdk::bitcoin::util::bip32;
+/// use bdk::bitcoin::bip32;
 /// use bdk::keys::{
 ///     any_network, DerivableKey, DescriptorKey, ExtendedKey, KeyError, ScriptContext,
 /// };
 ///
 /// struct MyCustomKeyType {
 ///     key_data: bitcoin::PrivateKey,
-///     chain_code: Vec<u8>,
+///     chain_code: [u8; 32],
 /// }
 ///
 /// impl<Ctx: ScriptContext> DerivableKey<Ctx> for MyCustomKeyType {
@@ -436,7 +436,7 @@ impl<Ctx: ScriptContext> From<bip32::ExtendedPrivKey> for ExtendedKey<Ctx> {
 ///             depth: 0,
 ///             parent_fingerprint: bip32::Fingerprint::default(),
 ///             private_key: self.key_data.inner,
-///             chain_code: bip32::ChainCode::from(self.chain_code.as_ref()),
+///             chain_code: bip32::ChainCode::from(&self.chain_code),
 ///             child_number: bip32::ChildNumber::Normal { index: 0 },
 ///         };
 ///
@@ -754,7 +754,7 @@ fn expand_multi_keys<Pk: IntoDescriptorKey<Ctx>, Ctx: ScriptContext>(
     let (key_map, valid_networks) = key_maps_networks.into_iter().fold(
         (KeyMap::default(), any_network()),
         |(mut keys_acc, net_acc), (key, net)| {
-            keys_acc.extend(key.into_iter());
+            keys_acc.extend(key);
             let net_acc = merge_networks(&net_acc, &net);
 
             (keys_acc, net_acc)
@@ -927,13 +927,22 @@ pub enum KeyError {
     Message(String),
 
     /// BIP32 error
-    Bip32(bitcoin::util::bip32::Error),
+    Bip32(bitcoin::bip32::Error),
     /// Miniscript error
     Miniscript(miniscript::Error),
 }
 
-impl_error!(miniscript::Error, Miniscript, KeyError);
-impl_error!(bitcoin::util::bip32::Error, Bip32, KeyError);
+impl From<miniscript::Error> for KeyError {
+    fn from(err: miniscript::Error) -> Self {
+        KeyError::Miniscript(err)
+    }
+}
+
+impl From<bip32::Error> for KeyError {
+    fn from(err: bip32::Error) -> Self {
+        KeyError::Bip32(err)
+    }
+}
 
 impl fmt::Display for KeyError {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
@@ -953,7 +962,7 @@ impl std::error::Error for KeyError {}
 
 #[cfg(test)]
 pub mod test {
-    use bitcoin::util::bip32;
+    use bitcoin::bip32;
 
     use super::*;
 

crates/bdk/src/lib.rs

@@ -17,9 +17,6 @@ extern crate std;
 pub extern crate alloc;
 
 pub extern crate bitcoin;
-#[cfg(feature = "hardware-signer")]
-pub extern crate hwi;
-extern crate log;
 pub extern crate miniscript;
 extern crate serde;
 extern crate serde_json;
@@ -27,9 +24,6 @@ extern crate serde_json;
 #[cfg(feature = "keys-bip39")]
 extern crate bip39;
 
-#[allow(unused_imports)]
-#[macro_use]
-pub(crate) mod error;
 pub mod descriptor;
 pub mod keys;
 pub mod psbt;
@@ -38,7 +32,6 @@ pub mod wallet;
 
 pub use descriptor::template;
 pub use descriptor::HdKeyPaths;
-pub use error::Error;
 pub use types::*;
 pub use wallet::signer;
 pub use wallet::signer::SignOptions;

crates/bdk/src/psbt/mod.rs

@@ -13,7 +13,7 @@
 
 use crate::FeeRate;
 use alloc::vec::Vec;
-use bitcoin::util::psbt::PartiallySignedTransaction as Psbt;
+use bitcoin::psbt::PartiallySignedTransaction as Psbt;
 use bitcoin::TxOut;
 
 // TODO upstream the functions here to `rust-bitcoin`?
@@ -35,24 +35,16 @@ pub trait PsbtUtils {
 }
 
 impl PsbtUtils for Psbt {
-    #[allow(clippy::all)] // We want to allow `manual_map` but it is too new.
     fn get_utxo_for(&self, input_index: usize) -> Option<TxOut> {
         let tx = &self.unsigned_tx;
+        let input = self.inputs.get(input_index)?;
 
-        if input_index >= tx.input.len() {
-            return None;
-        }
-
-        if let Some(input) = self.inputs.get(input_index) {
-            if let Some(wit_utxo) = &input.witness_utxo {
-                Some(wit_utxo.clone())
-            } else if let Some(in_tx) = &input.non_witness_utxo {
-                Some(in_tx.output[tx.input[input_index].previous_output.vout as usize].clone())
-            } else {
-                None
-            }
-        } else {
-            None
+        match (&input.witness_utxo, &input.non_witness_utxo) {
+            (Some(_), _) => input.witness_utxo.clone(),
+            (_, Some(_)) => input.non_witness_utxo.as_ref().map(|in_tx| {
+                in_tx.output[tx.input[input_index].previous_output.vout as usize].clone()
+            }),
+            _ => None,
         }
     }
 

crates/bdk/src/types.rs

@@ -14,8 +14,8 @@ use core::convert::AsRef;
 use core::ops::Sub;
 
 use bdk_chain::ConfirmationTime;
-use bitcoin::blockdata::transaction::{OutPoint, Transaction, TxOut};
-use bitcoin::{hash_types::Txid, util::psbt};
+use bitcoin::blockdata::transaction::{OutPoint, Sequence, TxOut};
+use bitcoin::{psbt, Weight};
 
 use serde::{Deserialize, Serialize};
 
@@ -99,8 +99,8 @@ impl FeeRate {
     }
 
     /// Calculate fee rate from `fee` and weight units (`wu`).
-    pub fn from_wu(fee: u64, wu: usize) -> FeeRate {
-        Self::from_vb(fee, wu.vbytes())
+    pub fn from_wu(fee: u64, wu: Weight) -> FeeRate {
+        Self::from_vb(fee, wu.to_vbytes_ceil() as usize)
     }
 
     /// Calculate fee rate from `fee` and `vbytes`.
@@ -120,8 +120,8 @@ impl FeeRate {
     }
 
     /// Calculate absolute fee in Satoshis using size in weight units.
-    pub fn fee_wu(&self, wu: usize) -> u64 {
-        self.fee_vb(wu.vbytes())
+    pub fn fee_wu(&self, wu: Weight) -> u64 {
+        self.fee_vb(wu.to_vbytes_ceil() as usize)
     }
 
     /// Calculate absolute fee in Satoshis using size in virtual bytes.
@@ -161,7 +161,7 @@ impl Vbytes for usize {
 ///
 /// [`Wallet`]: crate::Wallet
 #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Hash)]
-pub struct LocalUtxo {
+pub struct LocalOutput {
     /// Reference to a transaction output
     pub outpoint: OutPoint,
     /// Transaction output
@@ -192,11 +192,13 @@ pub struct WeightedUtxo {
 /// An unspent transaction output (UTXO).
 pub enum Utxo {
     /// A UTXO owned by the local wallet.
-    Local(LocalUtxo),
+    Local(LocalOutput),
     /// A UTXO owned by another wallet.
     Foreign {
         /// The location of the output.
         outpoint: OutPoint,
+        /// The nSequence value to set for this input.
+        sequence: Option<Sequence>,
         /// The information about the input we require to add it to a PSBT.
         // Box it to stop the type being too big.
         psbt_input: Box<psbt::Input>,
@@ -219,6 +221,7 @@ impl Utxo {
             Utxo::Foreign {
                 outpoint,
                 psbt_input,
+                ..
             } => {
                 if let Some(prev_tx) = &psbt_input.non_witness_utxo {
                     return &prev_tx.output[outpoint.vout as usize];
@@ -232,39 +235,13 @@ impl Utxo {
             }
         }
     }
-}
 
-/// A wallet transaction
-#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
-pub struct TransactionDetails {
-    /// Optional transaction
-    pub transaction: Option<Transaction>,
-    /// Transaction id
-    pub txid: Txid,
-    /// Received value (sats)
-    /// Sum of owned outputs of this transaction.
-    pub received: u64,
-    /// Sent value (sats)
-    /// Sum of owned inputs of this transaction.
-    pub sent: u64,
-    /// Fee value in sats if it was available.
-    pub fee: Option<u64>,
-    /// If the transaction is confirmed, contains height and Unix timestamp of the block containing the
-    /// transaction, unconfirmed transaction contains `None`.
-    pub confirmation_time: ConfirmationTime,
-}
-
-impl PartialOrd for TransactionDetails {
-    fn partial_cmp(&self, other: &Self) -> Option<core::cmp::Ordering> {
-        Some(self.cmp(other))
-    }
-}
-
-impl Ord for TransactionDetails {
-    fn cmp(&self, other: &Self) -> core::cmp::Ordering {
-        self.confirmation_time
-            .cmp(&other.confirmation_time)
-            .then_with(|| self.txid.cmp(&other.txid))
+    /// Get the sequence number if an explicit sequence number has to be set for this input.
+    pub fn sequence(&self) -> Option<Sequence> {
+        match self {
+            Utxo::Local(_) => None,
+            Utxo::Foreign { sequence, .. } => *sequence,
+        }
     }
 }
 

crates/bdk/src/wallet/coin_selection.rs

@@ -26,9 +26,12 @@
 //! ```
 //! # use std::str::FromStr;
 //! # use bitcoin::*;
-//! # use bdk::wallet::{self, coin_selection::*};
+//! # use bdk::wallet::{self, ChangeSet, coin_selection::*, coin_selection};
+//! # use bdk::wallet::error::CreateTxError;
+//! # use bdk_chain::PersistBackend;
 //! # use bdk::*;
 //! # use bdk::wallet::coin_selection::decide_change;
+//! # use anyhow::Error;
 //! # const TXIN_BASE_WEIGHT: usize = (32 + 4 + 4) * 4;
 //! #[derive(Debug)]
 //! struct AlwaysSpendEverything;
@@ -38,12 +41,12 @@
 //!         &self,
 //!         required_utxos: Vec<WeightedUtxo>,
 //!         optional_utxos: Vec<WeightedUtxo>,
-//!         fee_rate: FeeRate,
+//!         fee_rate: bdk::FeeRate,
 //!         target_amount: u64,
 //!         drain_script: &Script,
-//!     ) -> Result<CoinSelectionResult, bdk::Error> {
+//!     ) -> Result<CoinSelectionResult, coin_selection::Error> {
 //!         let mut selected_amount = 0;
-//!         let mut additional_weight = 0;
+//!         let mut additional_weight = Weight::ZERO;
 //!         let all_utxos_selected = required_utxos
 //!             .into_iter()
 //!             .chain(optional_utxos)
@@ -51,7 +54,9 @@
 //!                 (&mut selected_amount, &mut additional_weight),
 //!                 |(selected_amount, additional_weight), weighted_utxo| {
 //!                     **selected_amount += weighted_utxo.utxo.txout().value;
-//!                     **additional_weight += TXIN_BASE_WEIGHT + weighted_utxo.satisfaction_weight;
+//!                     **additional_weight += Weight::from_wu(
+//!                         (TXIN_BASE_WEIGHT + weighted_utxo.satisfaction_weight) as u64,
+//!                     );
 //!                     Some(weighted_utxo.utxo)
 //!                 },
 //!             )
@@ -59,7 +64,7 @@
 //!         let additional_fees = fee_rate.fee_wu(additional_weight);
 //!         let amount_needed_with_fees = additional_fees + target_amount;
 //!         if selected_amount < amount_needed_with_fees {
-//!             return Err(bdk::Error::InsufficientFunds {
+//!             return Err(coin_selection::Error::InsufficientFunds {
 //!                 needed: amount_needed_with_fees,
 //!                 available: selected_amount,
 //!             });
@@ -80,8 +85,11 @@
 //! # let mut wallet = doctest_wallet!();
 //! // create wallet, sync, ...
 //!
-//! let to_address = Address::from_str("2N4eQYCbKUHCCTUjBJeHcJp9ok6J2GZsTDt").unwrap();
-//! let (psbt, details) = {
+//! let to_address = Address::from_str("2N4eQYCbKUHCCTUjBJeHcJp9ok6J2GZsTDt")
+//!     .unwrap()
+//!     .require_network(Network::Testnet)
+//!     .unwrap();
+//! let psbt = {
 //!     let mut builder = wallet.build_tx().coin_selection(AlwaysSpendEverything);
 //!     builder.add_recipient(to_address.script_pubkey(), 50_000);
 //!     builder.finish()?
@@ -89,19 +97,22 @@
 //!
 //! // inspect, sign, broadcast, ...
 //!
-//! # Ok::<(), bdk::Error>(())
+//! # Ok::<(), anyhow::Error>(())
 //! ```
 
+use crate::chain::collections::HashSet;
 use crate::types::FeeRate;
 use crate::wallet::utils::IsDust;
+use crate::Utxo;
 use crate::WeightedUtxo;
-use crate::{error::Error, Utxo};
 
 use alloc::vec::Vec;
 use bitcoin::consensus::encode::serialize;
-use bitcoin::Script;
+use bitcoin::OutPoint;
+use bitcoin::{Script, Weight};
 
 use core::convert::TryInto;
+use core::fmt::{self, Formatter};
 use rand::seq::SliceRandom;
 
 /// Default coin selection algorithm used by [`TxBuilder`](super::tx_builder::TxBuilder) if not
@@ -112,6 +123,43 @@ pub type DefaultCoinSelectionAlgorithm = BranchAndBoundCoinSelection;
 // prev_txid (32 bytes) + prev_vout (4 bytes) + sequence (4 bytes)
 pub(crate) const TXIN_BASE_WEIGHT: usize = (32 + 4 + 4) * 4;
 
+/// Errors that can be thrown by the [`coin_selection`](crate::wallet::coin_selection) module
+#[derive(Debug)]
+pub enum Error {
+    /// Wallet's UTXO set is not enough to cover recipient's requested plus fee
+    InsufficientFunds {
+        /// Sats needed for some transaction
+        needed: u64,
+        /// Sats available for spending
+        available: u64,
+    },
+    /// Branch and bound coin selection tries to avoid needing a change by finding the right inputs for
+    /// the desired outputs plus fee, if there is not such combination this error is thrown
+    BnBNoExactMatch,
+    /// Branch and bound coin selection possible attempts with sufficiently big UTXO set could grow
+    /// exponentially, thus a limit is set, and when hit, this error is thrown
+    BnBTotalTriesExceeded,
+}
+
+impl fmt::Display for Error {
+    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
+        match self {
+            Self::InsufficientFunds { needed, available } => write!(
+                f,
+                "Insufficient funds: {} sat available of {} sat needed",
+                available, needed
+            ),
+            Self::BnBTotalTriesExceeded => {
+                write!(f, "Branch and bound coin selection: total tries exceeded")
+            }
+            Self::BnBNoExactMatch => write!(f, "Branch and bound coin selection: not exact match"),
+        }
+    }
+}
+
+#[cfg(feature = "std")]
+impl std::error::Error for Error {}
+
 #[derive(Debug)]
 /// Remaining amount after performing coin selection
 pub enum Excess {
@@ -208,12 +256,6 @@ impl CoinSelectionAlgorithm for LargestFirstCoinSelection {
         target_amount: u64,
         drain_script: &Script,
     ) -> Result<CoinSelectionResult, Error> {
-        log::debug!(
-            "target_amount = `{}`, fee_rate = `{:?}`",
-            target_amount,
-            fee_rate
-        );
-
         // We put the "required UTXOs" first and make sure the optional UTXOs are sorted,
         // initially smallest to largest, before being reversed with `.rev()`.
         let utxos = {
@@ -302,16 +344,10 @@ fn select_sorted_utxos(
             (&mut selected_amount, &mut fee_amount),
             |(selected_amount, fee_amount), (must_use, weighted_utxo)| {
                 if must_use || **selected_amount < target_amount + **fee_amount {
-                    **fee_amount +=
-                        fee_rate.fee_wu(TXIN_BASE_WEIGHT + weighted_utxo.satisfaction_weight);
+                    **fee_amount += fee_rate.fee_wu(Weight::from_wu(
+                        (TXIN_BASE_WEIGHT + weighted_utxo.satisfaction_weight) as u64,
+                    ));
                     **selected_amount += weighted_utxo.utxo.txout().value;
-
-                    log::debug!(
-                        "Selected {}, updated fee_amount = `{}`",
-                        weighted_utxo.utxo.outpoint(),
-                        fee_amount
-                    );
-
                     Some(weighted_utxo.utxo)
                 } else {
                     None
@@ -351,7 +387,9 @@ struct OutputGroup {
 
 impl OutputGroup {
     fn new(weighted_utxo: WeightedUtxo, fee_rate: FeeRate) -> Self {
-        let fee = fee_rate.fee_wu(TXIN_BASE_WEIGHT + weighted_utxo.satisfaction_weight);
+        let fee = fee_rate.fee_wu(Weight::from_wu(
+            (TXIN_BASE_WEIGHT + weighted_utxo.satisfaction_weight) as u64,
+        ));
         let effective_value = weighted_utxo.utxo.txout().value as i64 - fee as i64;
         OutputGroup {
             weighted_utxo,
@@ -675,16 +713,36 @@ impl BranchAndBoundCoinSelection {
     }
 }
 
+/// Remove duplicate UTXOs.
+///
+/// If a UTXO appears in both `required` and `optional`, the appearance in `required` is kept.
+pub(crate) fn filter_duplicates<I>(required: I, optional: I) -> (I, I)
+where
+    I: IntoIterator<Item = WeightedUtxo> + FromIterator<WeightedUtxo>,
+{
+    let mut visited = HashSet::<OutPoint>::new();
+    let required = required
+        .into_iter()
+        .filter(|utxo| visited.insert(utxo.utxo.outpoint()))
+        .collect::<I>();
+    let optional = optional
+        .into_iter()
+        .filter(|utxo| visited.insert(utxo.utxo.outpoint()))
+        .collect::<I>();
+    (required, optional)
+}
+
 #[cfg(test)]
 mod test {
     use assert_matches::assert_matches;
     use core::str::FromStr;
 
     use bdk_chain::ConfirmationTime;
-    use bitcoin::{OutPoint, Script, TxOut};
+    use bitcoin::{OutPoint, ScriptBuf, TxOut};
 
     use super::*;
     use crate::types::*;
+    use crate::wallet::coin_selection::filter_duplicates;
     use crate::wallet::Vbytes;
 
     use rand::rngs::StdRng;
@@ -706,11 +764,11 @@ mod test {
         .unwrap();
         WeightedUtxo {
             satisfaction_weight: P2WPKH_SATISFACTION_SIZE,
-            utxo: Utxo::Local(LocalUtxo {
+            utxo: Utxo::Local(LocalOutput {
                 outpoint,
                 txout: TxOut {
                     value,
-                    script_pubkey: Script::new(),
+                    script_pubkey: ScriptBuf::new(),
                 },
                 keychain: KeychainKind::External,
                 is_spent: false,
@@ -763,17 +821,18 @@ mod test {
 
     fn generate_random_utxos(rng: &mut StdRng, utxos_number: usize) -> Vec<WeightedUtxo> {
         let mut res = Vec::new();
-        for _ in 0..utxos_number {
+        for i in 0..utxos_number {
             res.push(WeightedUtxo {
                 satisfaction_weight: P2WPKH_SATISFACTION_SIZE,
-                utxo: Utxo::Local(LocalUtxo {
-                    outpoint: OutPoint::from_str(
-                        "ebd9813ecebc57ff8f30797de7c205e3c7498ca950ea4341ee51a685ff2fa30a:0",
-                    )
+                utxo: Utxo::Local(LocalOutput {
+                    outpoint: OutPoint::from_str(&format!(
+                        "ebd9813ecebc57ff8f30797de7c205e3c7498ca950ea4341ee51a685ff2fa30a:{}",
+                        i
+                    ))
                     .unwrap(),
                     txout: TxOut {
                         value: rng.gen_range(0..200000000),
-                        script_pubkey: Script::new(),
+                        script_pubkey: ScriptBuf::new(),
                     },
                     keychain: KeychainKind::External,
                     is_spent: false,
@@ -793,24 +852,26 @@ mod test {
     }
 
     fn generate_same_value_utxos(utxos_value: u64, utxos_number: usize) -> Vec<WeightedUtxo> {
-        let utxo = WeightedUtxo {
-            satisfaction_weight: P2WPKH_SATISFACTION_SIZE,
-            utxo: Utxo::Local(LocalUtxo {
-                outpoint: OutPoint::from_str(
-                    "ebd9813ecebc57ff8f30797de7c205e3c7498ca950ea4341ee51a685ff2fa30a:0",
-                )
-                .unwrap(),
-                txout: TxOut {
-                    value: utxos_value,
-                    script_pubkey: Script::new(),
-                },
-                keychain: KeychainKind::External,
-                is_spent: false,
-                derivation_index: 42,
-                confirmation_time: ConfirmationTime::Unconfirmed { last_seen: 0 },
-            }),
-        };
-        vec![utxo; utxos_number]
+        (0..utxos_number)
+            .map(|i| WeightedUtxo {
+                satisfaction_weight: P2WPKH_SATISFACTION_SIZE,
+                utxo: Utxo::Local(LocalOutput {
+                    outpoint: OutPoint::from_str(&format!(
+                        "ebd9813ecebc57ff8f30797de7c205e3c7498ca950ea4341ee51a685ff2fa30a:{}",
+                        i
+                    ))
+                    .unwrap(),
+                    txout: TxOut {
+                        value: utxos_value,
+                        script_pubkey: ScriptBuf::new(),
+                    },
+                    keychain: KeychainKind::External,
+                    is_spent: false,
+                    derivation_index: 42,
+                    confirmation_time: ConfirmationTime::Unconfirmed { last_seen: 0 },
+                }),
+            })
+            .collect()
     }
 
     fn sum_random_utxos(mut rng: &mut StdRng, utxos: &mut Vec<WeightedUtxo>) -> u64 {
@@ -825,10 +886,10 @@ mod test {
     #[test]
     fn test_largest_first_coin_selection_success() {
         let utxos = get_test_utxos();
-        let drain_script = Script::default();
+        let drain_script = ScriptBuf::default();
         let target_amount = 250_000 + FEE_AMOUNT;
 
-        let result = LargestFirstCoinSelection::default()
+        let result = LargestFirstCoinSelection
             .coin_select(
                 utxos,
                 vec![],
@@ -846,10 +907,10 @@ mod test {
     #[test]
     fn test_largest_first_coin_selection_use_all() {
         let utxos = get_test_utxos();
-        let drain_script = Script::default();
+        let drain_script = ScriptBuf::default();
         let target_amount = 20_000 + FEE_AMOUNT;
 
-        let result = LargestFirstCoinSelection::default()
+        let result = LargestFirstCoinSelection
             .coin_select(
                 utxos,
                 vec![],
@@ -867,10 +928,10 @@ mod test {
     #[test]
     fn test_largest_first_coin_selection_use_only_necessary() {
         let utxos = get_test_utxos();
-        let drain_script = Script::default();
+        let drain_script = ScriptBuf::default();
         let target_amount = 20_000 + FEE_AMOUNT;
 
-        let result = LargestFirstCoinSelection::default()
+        let result = LargestFirstCoinSelection
             .coin_select(
                 vec![],
                 utxos,
@@ -889,10 +950,10 @@ mod test {
     #[should_panic(expected = "InsufficientFunds")]
     fn test_largest_first_coin_selection_insufficient_funds() {
         let utxos = get_test_utxos();
-        let drain_script = Script::default();
+        let drain_script = ScriptBuf::default();
         let target_amount = 500_000 + FEE_AMOUNT;
 
-        LargestFirstCoinSelection::default()
+        LargestFirstCoinSelection
             .coin_select(
                 vec![],
                 utxos,
@@ -907,10 +968,10 @@ mod test {
     #[should_panic(expected = "InsufficientFunds")]
     fn test_largest_first_coin_selection_insufficient_funds_high_fees() {
         let utxos = get_test_utxos();
-        let drain_script = Script::default();
+        let drain_script = ScriptBuf::default();
         let target_amount = 250_000 + FEE_AMOUNT;
 
-        LargestFirstCoinSelection::default()
+        LargestFirstCoinSelection
             .coin_select(
                 vec![],
                 utxos,
@@ -924,10 +985,10 @@ mod test {
     #[test]
     fn test_oldest_first_coin_selection_success() {
         let utxos = get_oldest_first_test_utxos();
-        let drain_script = Script::default();
+        let drain_script = ScriptBuf::default();
         let target_amount = 180_000 + FEE_AMOUNT;
 
-        let result = OldestFirstCoinSelection::default()
+        let result = OldestFirstCoinSelection
             .coin_select(
                 vec![],
                 utxos,
@@ -945,10 +1006,10 @@ mod test {
     #[test]
     fn test_oldest_first_coin_selection_use_all() {
         let utxos = get_oldest_first_test_utxos();
-        let drain_script = Script::default();
+        let drain_script = ScriptBuf::default();
         let target_amount = 20_000 + FEE_AMOUNT;
 
-        let result = OldestFirstCoinSelection::default()
+        let result = OldestFirstCoinSelection
             .coin_select(
                 utxos,
                 vec![],
@@ -966,10 +1027,10 @@ mod test {
     #[test]
     fn test_oldest_first_coin_selection_use_only_necessary() {
         let utxos = get_oldest_first_test_utxos();
-        let drain_script = Script::default();
+        let drain_script = ScriptBuf::default();
         let target_amount = 20_000 + FEE_AMOUNT;
 
-        let result = OldestFirstCoinSelection::default()
+        let result = OldestFirstCoinSelection
             .coin_select(
                 vec![],
                 utxos,
@@ -988,10 +1049,10 @@ mod test {
     #[should_panic(expected = "InsufficientFunds")]
     fn test_oldest_first_coin_selection_insufficient_funds() {
         let utxos = get_oldest_first_test_utxos();
-        let drain_script = Script::default();
+        let drain_script = ScriptBuf::default();
         let target_amount = 600_000 + FEE_AMOUNT;
 
-        OldestFirstCoinSelection::default()
+        OldestFirstCoinSelection
             .coin_select(
                 vec![],
                 utxos,
@@ -1008,9 +1069,9 @@ mod test {
         let utxos = get_oldest_first_test_utxos();
 
         let target_amount: u64 = utxos.iter().map(|wu| wu.utxo.txout().value).sum::<u64>() - 50;
-        let drain_script = Script::default();
+        let drain_script = ScriptBuf::default();
 
-        OldestFirstCoinSelection::default()
+        OldestFirstCoinSelection
             .coin_select(
                 vec![],
                 utxos,
@@ -1027,7 +1088,7 @@ mod test {
         // select three outputs
         let utxos = generate_same_value_utxos(100_000, 20);
 
-        let drain_script = Script::default();
+        let drain_script = ScriptBuf::default();
 
         let target_amount = 250_000 + FEE_AMOUNT;
 
@@ -1049,7 +1110,7 @@ mod test {
     #[test]
     fn test_bnb_coin_selection_required_are_enough() {
         let utxos = get_test_utxos();
-        let drain_script = Script::default();
+        let drain_script = ScriptBuf::default();
         let target_amount = 20_000 + FEE_AMOUNT;
 
         let result = BranchAndBoundCoinSelection::default()
@@ -1070,7 +1131,7 @@ mod test {
     #[test]
     fn test_bnb_coin_selection_optional_are_enough() {
         let utxos = get_test_utxos();
-        let drain_script = Script::default();
+        let drain_script = ScriptBuf::default();
         let target_amount = 299756 + FEE_AMOUNT;
 
         let result = BranchAndBoundCoinSelection::default()
@@ -1106,7 +1167,7 @@ mod test {
         assert_eq!(amount, 100_000);
         let amount: u64 = optional.iter().map(|u| u.utxo.txout().value).sum();
         assert!(amount > 150_000);
-        let drain_script = Script::default();
+        let drain_script = ScriptBuf::default();
 
         let target_amount = 150_000 + FEE_AMOUNT;
 
@@ -1129,7 +1190,7 @@ mod test {
     #[should_panic(expected = "InsufficientFunds")]
     fn test_bnb_coin_selection_insufficient_funds() {
         let utxos = get_test_utxos();
-        let drain_script = Script::default();
+        let drain_script = ScriptBuf::default();
         let target_amount = 500_000 + FEE_AMOUNT;
 
         BranchAndBoundCoinSelection::default()
@@ -1147,7 +1208,7 @@ mod test {
     #[should_panic(expected = "InsufficientFunds")]
     fn test_bnb_coin_selection_insufficient_funds_high_fees() {
         let utxos = get_test_utxos();
-        let drain_script = Script::default();
+        let drain_script = ScriptBuf::default();
         let target_amount = 250_000 + FEE_AMOUNT;
 
         BranchAndBoundCoinSelection::default()
@@ -1164,7 +1225,7 @@ mod test {
     #[test]
     fn test_bnb_coin_selection_check_fee_rate() {
         let utxos = get_test_utxos();
-        let drain_script = Script::default();
+        let drain_script = ScriptBuf::default();
         let target_amount = 99932; // first utxo's effective value
 
         let result = BranchAndBoundCoinSelection::new(0)
@@ -1192,7 +1253,7 @@ mod test {
         for _i in 0..200 {
             let mut optional_utxos = generate_random_utxos(&mut rng, 16);
             let target_amount = sum_random_utxos(&mut rng, &mut optional_utxos);
-            let drain_script = Script::default();
+            let drain_script = ScriptBuf::default();
             let result = BranchAndBoundCoinSelection::new(0)
                 .coin_select(
                     vec![],
@@ -1220,7 +1281,7 @@ mod test {
         let size_of_change = 31;
         let cost_of_change = size_of_change as f32 * fee_rate.as_sat_per_vb();
 
-        let drain_script = Script::default();
+        let drain_script = ScriptBuf::default();
         let target_amount = 20_000 + FEE_AMOUNT;
         BranchAndBoundCoinSelection::new(size_of_change)
             .bnb(
@@ -1251,7 +1312,7 @@ mod test {
         let cost_of_change = size_of_change as f32 * fee_rate.as_sat_per_vb();
         let target_amount = 20_000 + FEE_AMOUNT;
 
-        let drain_script = Script::default();
+        let drain_script = ScriptBuf::default();
 
         BranchAndBoundCoinSelection::new(size_of_change)
             .bnb(
@@ -1287,7 +1348,7 @@ mod test {
         // cost_of_change + 5.
         let target_amount = 2 * 50_000 - 2 * 67 - cost_of_change.ceil() as i64 + 5;
 
-        let drain_script = Script::default();
+        let drain_script = ScriptBuf::default();
 
         let result = BranchAndBoundCoinSelection::new(size_of_change)
             .bnb(
@@ -1327,7 +1388,7 @@ mod test {
             let target_amount =
                 optional_utxos[3].effective_value + optional_utxos[23].effective_value;
 
-            let drain_script = Script::default();
+            let drain_script = ScriptBuf::default();
 
             let result = BranchAndBoundCoinSelection::new(0)
                 .bnb(
@@ -1358,7 +1419,7 @@ mod test {
             .map(|u| OutputGroup::new(u, fee_rate))
             .collect();
 
-        let drain_script = Script::default();
+        let drain_script = ScriptBuf::default();
 
         let result = BranchAndBoundCoinSelection::default().single_random_draw(
             vec![],
@@ -1376,7 +1437,7 @@ mod test {
     #[test]
     fn test_bnb_exclude_negative_effective_value() {
         let utxos = get_test_utxos();
-        let drain_script = Script::default();
+        let drain_script = ScriptBuf::default();
 
         let selection = BranchAndBoundCoinSelection::default().coin_select(
             vec![],
@@ -1398,7 +1459,7 @@ mod test {
     #[test]
     fn test_bnb_include_negative_effective_value_when_required() {
         let utxos = get_test_utxos();
-        let drain_script = Script::default();
+        let drain_script = ScriptBuf::default();
 
         let (required, optional) = utxos
             .into_iter()
@@ -1424,7 +1485,7 @@ mod test {
     #[test]
     fn test_bnb_sum_of_effective_value_negative() {
         let utxos = get_test_utxos();
-        let drain_script = Script::default();
+        let drain_script = ScriptBuf::default();
 
         let selection = BranchAndBoundCoinSelection::default().coin_select(
             utxos,
@@ -1442,4 +1503,95 @@ mod test {
             })
         );
     }
+
+    #[test]
+    fn test_filter_duplicates() {
+        fn utxo(txid: &str, value: u64) -> WeightedUtxo {
+            WeightedUtxo {
+                satisfaction_weight: 0,
+                utxo: Utxo::Local(LocalOutput {
+                    outpoint: OutPoint::new(bitcoin::hashes::Hash::hash(txid.as_bytes()), 0),
+                    txout: TxOut {
+                        value,
+                        script_pubkey: ScriptBuf::new(),
+                    },
+                    keychain: KeychainKind::External,
+                    is_spent: false,
+                    derivation_index: 0,
+                    confirmation_time: ConfirmationTime::Confirmed {
+                        height: 12345,
+                        time: 12345,
+                    },
+                }),
+            }
+        }
+
+        fn to_utxo_vec(utxos: &[(&str, u64)]) -> Vec<WeightedUtxo> {
+            let mut v = utxos
+                .iter()
+                .map(|&(txid, value)| utxo(txid, value))
+                .collect::<Vec<_>>();
+            v.sort_by_key(|u| u.utxo.outpoint());
+            v
+        }
+
+        struct TestCase<'a> {
+            name: &'a str,
+            required: &'a [(&'a str, u64)],
+            optional: &'a [(&'a str, u64)],
+            exp_required: &'a [(&'a str, u64)],
+            exp_optional: &'a [(&'a str, u64)],
+        }
+
+        let test_cases = [
+            TestCase {
+                name: "no_duplicates",
+                required: &[("A", 1000), ("B", 2100)],
+                optional: &[("C", 1000)],
+                exp_required: &[("A", 1000), ("B", 2100)],
+                exp_optional: &[("C", 1000)],
+            },
+            TestCase {
+                name: "duplicate_required_utxos",
+                required: &[("A", 3000), ("B", 1200), ("C", 1234), ("A", 3000)],
+                optional: &[("D", 2100)],
+                exp_required: &[("A", 3000), ("B", 1200), ("C", 1234)],
+                exp_optional: &[("D", 2100)],
+            },
+            TestCase {
+                name: "duplicate_optional_utxos",
+                required: &[("A", 3000), ("B", 1200)],
+                optional: &[("C", 5000), ("D", 1300), ("C", 5000)],
+                exp_required: &[("A", 3000), ("B", 1200)],
+                exp_optional: &[("C", 5000), ("D", 1300)],
+            },
+            TestCase {
+                name: "duplicate_across_required_and_optional_utxos",
+                required: &[("A", 3000), ("B", 1200), ("C", 2100)],
+                optional: &[("A", 3000), ("D", 1200), ("E", 5000)],
+                exp_required: &[("A", 3000), ("B", 1200), ("C", 2100)],
+                exp_optional: &[("D", 1200), ("E", 5000)],
+            },
+        ];
+
+        for (i, t) in test_cases.into_iter().enumerate() {
+            println!("Case {}: {}", i, t.name);
+            let (required, optional) =
+                filter_duplicates(to_utxo_vec(t.required), to_utxo_vec(t.optional));
+            assert_eq!(
+                required,
+                to_utxo_vec(t.exp_required),
+                "[{}:{}] unexpected `required` result",
+                i,
+                t.name
+            );
+            assert_eq!(
+                optional,
+                to_utxo_vec(t.exp_optional),
+                "[{}:{}] unexpected `optional` result",
+                i,
+                t.name
+            );
+        }
+    }
 }

crates/bdk/src/wallet/error.rs

@@ -0,0 +1,292 @@
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
+//
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
+//
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
+
+//! Errors that can be thrown by the [`Wallet`](crate::wallet::Wallet)
+
+use crate::descriptor::policy::PolicyError;
+use crate::descriptor::DescriptorError;
+use crate::wallet::coin_selection;
+use crate::{descriptor, FeeRate, KeychainKind};
+use alloc::string::String;
+use bitcoin::{absolute, psbt, OutPoint, Sequence, Txid};
+use core::fmt;
+
+/// Errors returned by miniscript when updating inconsistent PSBTs
+#[derive(Debug, Clone)]
+pub enum MiniscriptPsbtError {
+    /// Descriptor key conversion error
+    Conversion(miniscript::descriptor::ConversionError),
+    /// Return error type for PsbtExt::update_input_with_descriptor
+    UtxoUpdate(miniscript::psbt::UtxoUpdateError),
+    /// Return error type for PsbtExt::update_output_with_descriptor
+    OutputUpdate(miniscript::psbt::OutputUpdateError),
+}
+
+impl fmt::Display for MiniscriptPsbtError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Self::Conversion(err) => write!(f, "Conversion error: {}", err),
+            Self::UtxoUpdate(err) => write!(f, "UTXO update error: {}", err),
+            Self::OutputUpdate(err) => write!(f, "Output update error: {}", err),
+        }
+    }
+}
+
+#[cfg(feature = "std")]
+impl std::error::Error for MiniscriptPsbtError {}
+
+#[derive(Debug)]
+/// Error returned from [`TxBuilder::finish`]
+///
+/// [`TxBuilder::finish`]: crate::wallet::tx_builder::TxBuilder::finish
+pub enum CreateTxError<P> {
+    /// There was a problem with the descriptors passed in
+    Descriptor(DescriptorError),
+    /// We were unable to write wallet data to the persistence backend
+    Persist(P),
+    /// There was a problem while extracting and manipulating policies
+    Policy(PolicyError),
+    /// Spending policy is not compatible with this [`KeychainKind`]
+    SpendingPolicyRequired(KeychainKind),
+    /// Requested invalid transaction version '0'
+    Version0,
+    /// Requested transaction version `1`, but at least `2` is needed to use OP_CSV
+    Version1Csv,
+    /// Requested `LockTime` is less than is required to spend from this script
+    LockTime {
+        /// Requested `LockTime`
+        requested: absolute::LockTime,
+        /// Required `LockTime`
+        required: absolute::LockTime,
+    },
+    /// Cannot enable RBF with a `Sequence` >= 0xFFFFFFFE
+    RbfSequence,
+    /// Cannot enable RBF with `Sequence` given a required OP_CSV
+    RbfSequenceCsv {
+        /// Given RBF `Sequence`
+        rbf: Sequence,
+        /// Required OP_CSV `Sequence`
+        csv: Sequence,
+    },
+    /// When bumping a tx the absolute fee requested is lower than replaced tx absolute fee
+    FeeTooLow {
+        /// Required fee absolute value (satoshi)
+        required: u64,
+    },
+    /// When bumping a tx the fee rate requested is lower than required
+    FeeRateTooLow {
+        /// Required fee rate (satoshi/vbyte)
+        required: FeeRate,
+    },
+    /// `manually_selected_only` option is selected but no utxo has been passed
+    NoUtxosSelected,
+    /// Output created is under the dust limit, 546 satoshis
+    OutputBelowDustLimit(usize),
+    /// The `change_policy` was set but the wallet does not have a change_descriptor
+    ChangePolicyDescriptor,
+    /// There was an error with coin selection
+    CoinSelection(coin_selection::Error),
+    /// Wallet's UTXO set is not enough to cover recipient's requested plus fee
+    InsufficientFunds {
+        /// Sats needed for some transaction
+        needed: u64,
+        /// Sats available for spending
+        available: u64,
+    },
+    /// Cannot build a tx without recipients
+    NoRecipients,
+    /// Partially signed bitcoin transaction error
+    Psbt(psbt::Error),
+    /// In order to use the [`TxBuilder::add_global_xpubs`] option every extended
+    /// key in the descriptor must either be a master key itself (having depth = 0) or have an
+    /// explicit origin provided
+    ///
+    /// [`TxBuilder::add_global_xpubs`]: crate::wallet::tx_builder::TxBuilder::add_global_xpubs
+    MissingKeyOrigin(String),
+    /// Happens when trying to spend an UTXO that is not in the internal database
+    UnknownUtxo,
+    /// Missing non_witness_utxo on foreign utxo for given `OutPoint`
+    MissingNonWitnessUtxo(OutPoint),
+    /// Miniscript PSBT error
+    MiniscriptPsbt(MiniscriptPsbtError),
+}
+
+impl<P> fmt::Display for CreateTxError<P>
+where
+    P: fmt::Display,
+{
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Self::Descriptor(e) => e.fmt(f),
+            Self::Persist(e) => {
+                write!(
+                    f,
+                    "failed to write wallet data to persistence backend: {}",
+                    e
+                )
+            }
+            Self::Policy(e) => e.fmt(f),
+            CreateTxError::SpendingPolicyRequired(keychain_kind) => {
+                write!(f, "Spending policy required: {:?}", keychain_kind)
+            }
+            CreateTxError::Version0 => {
+                write!(f, "Invalid version `0`")
+            }
+            CreateTxError::Version1Csv => {
+                write!(
+                    f,
+                    "TxBuilder requested version `1`, but at least `2` is needed to use OP_CSV"
+                )
+            }
+            CreateTxError::LockTime {
+                requested,
+                required,
+            } => {
+                write!(f, "TxBuilder requested timelock of `{:?}`, but at least `{:?}` is required to spend from this script", required, requested)
+            }
+            CreateTxError::RbfSequence => {
+                write!(f, "Cannot enable RBF with a nSequence >= 0xFFFFFFFE")
+            }
+            CreateTxError::RbfSequenceCsv { rbf, csv } => {
+                write!(
+                    f,
+                    "Cannot enable RBF with nSequence `{:?}` given a required OP_CSV of `{:?}`",
+                    rbf, csv
+                )
+            }
+            CreateTxError::FeeTooLow { required } => {
+                write!(f, "Fee to low: required {} sat", required)
+            }
+            CreateTxError::FeeRateTooLow { required } => {
+                write!(
+                    f,
+                    "Fee rate too low: required {} sat/vbyte",
+                    required.as_sat_per_vb()
+                )
+            }
+            CreateTxError::NoUtxosSelected => {
+                write!(f, "No UTXO selected")
+            }
+            CreateTxError::OutputBelowDustLimit(limit) => {
+                write!(f, "Output below the dust limit: {}", limit)
+            }
+            CreateTxError::ChangePolicyDescriptor => {
+                write!(
+                    f,
+                    "The `change_policy` can be set only if the wallet has a change_descriptor"
+                )
+            }
+            CreateTxError::CoinSelection(e) => e.fmt(f),
+            CreateTxError::InsufficientFunds { needed, available } => {
+                write!(
+                    f,
+                    "Insufficient funds: {} sat available of {} sat needed",
+                    available, needed
+                )
+            }
+            CreateTxError::NoRecipients => {
+                write!(f, "Cannot build tx without recipients")
+            }
+            CreateTxError::Psbt(e) => e.fmt(f),
+            CreateTxError::MissingKeyOrigin(err) => {
+                write!(f, "Missing key origin: {}", err)
+            }
+            CreateTxError::UnknownUtxo => {
+                write!(f, "UTXO not found in the internal database")
+            }
+            CreateTxError::MissingNonWitnessUtxo(outpoint) => {
+                write!(f, "Missing non_witness_utxo on foreign utxo {}", outpoint)
+            }
+            CreateTxError::MiniscriptPsbt(err) => {
+                write!(f, "Miniscript PSBT error: {}", err)
+            }
+        }
+    }
+}
+
+impl<P> From<descriptor::error::Error> for CreateTxError<P> {
+    fn from(err: descriptor::error::Error) -> Self {
+        CreateTxError::Descriptor(err)
+    }
+}
+
+impl<P> From<PolicyError> for CreateTxError<P> {
+    fn from(err: PolicyError) -> Self {
+        CreateTxError::Policy(err)
+    }
+}
+
+impl<P> From<MiniscriptPsbtError> for CreateTxError<P> {
+    fn from(err: MiniscriptPsbtError) -> Self {
+        CreateTxError::MiniscriptPsbt(err)
+    }
+}
+
+impl<P> From<psbt::Error> for CreateTxError<P> {
+    fn from(err: psbt::Error) -> Self {
+        CreateTxError::Psbt(err)
+    }
+}
+
+impl<P> From<coin_selection::Error> for CreateTxError<P> {
+    fn from(err: coin_selection::Error) -> Self {
+        CreateTxError::CoinSelection(err)
+    }
+}
+
+#[cfg(feature = "std")]
+impl<P: core::fmt::Display + core::fmt::Debug> std::error::Error for CreateTxError<P> {}
+
+#[derive(Debug)]
+/// Error returned from [`Wallet::build_fee_bump`]
+///
+/// [`Wallet::build_fee_bump`]: super::Wallet::build_fee_bump
+pub enum BuildFeeBumpError {
+    /// Happens when trying to spend an UTXO that is not in the internal database
+    UnknownUtxo(OutPoint),
+    /// Thrown when a tx is not found in the internal database
+    TransactionNotFound(Txid),
+    /// Happens when trying to bump a transaction that is already confirmed
+    TransactionConfirmed(Txid),
+    /// Trying to replace a tx that has a sequence >= `0xFFFFFFFE`
+    IrreplaceableTransaction(Txid),
+    /// Node doesn't have data to estimate a fee rate
+    FeeRateUnavailable,
+}
+
+impl fmt::Display for BuildFeeBumpError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Self::UnknownUtxo(outpoint) => write!(
+                f,
+                "UTXO not found in the internal database with txid: {}, vout: {}",
+                outpoint.txid, outpoint.vout
+            ),
+            Self::TransactionNotFound(txid) => {
+                write!(
+                    f,
+                    "Transaction not found in the internal database with txid: {}",
+                    txid
+                )
+            }
+            Self::TransactionConfirmed(txid) => {
+                write!(f, "Transaction already confirmed with txid: {}", txid)
+            }
+            Self::IrreplaceableTransaction(txid) => {
+                write!(f, "Transaction can't be replaced with txid: {}", txid)
+            }
+            Self::FeeRateUnavailable => write!(f, "Fee rate unavailable"),
+        }
+    }
+}
+
+#[cfg(feature = "std")]
+impl std::error::Error for BuildFeeBumpError {}

crates/bdk/src/wallet/export.rs

@@ -126,13 +126,12 @@ impl FullyNodedExport {
         Self::is_compatible_with_core(&descriptor)?;
 
         let blockheight = if include_blockheight {
-            wallet
-                .transactions()
-                .next()
-                .map_or(0, |canonical_tx| match canonical_tx.observed_as {
+            wallet.transactions().next().map_or(0, |canonical_tx| {
+                match canonical_tx.chain_position {
                     bdk_chain::ChainPosition::Confirmed(a) => a.confirmation_height,
                     bdk_chain::ChainPosition::Unconfirmed(_) => 0,
-                })
+                }
+            })
         } else {
             0
         };
@@ -232,7 +231,7 @@ mod test {
             input: vec![],
             output: vec![],
             version: 0,
-            lock_time: bitcoin::PackedLockTime::ZERO,
+            lock_time: bitcoin::absolute::LockTime::ZERO,
         };
         wallet
             .insert_checkpoint(BlockId {

crates/bdk/src/wallet/hardwaresigner.rs

@@ -19,7 +19,7 @@
 //! # use bdk::wallet::hardwaresigner::HWISigner;
 //! # use bdk::wallet::AddressIndex::New;
 //! # use bdk::{FeeRate, KeychainKind, SignOptions, Wallet};
-//! # use hwi::{types::HWIChain, HWIClient};
+//! # use hwi::HWIClient;
 //! # use std::sync::Arc;
 //! #
 //! # fn main() -> Result<(), Box<dyn std::error::Error>> {
@@ -28,7 +28,7 @@
 //!     panic!("No devices found!");
 //! }
 //! let first_device = devices.remove(0)?;
-//! let custom_signer = HWISigner::from_device(&first_device, HWIChain::Test)?;
+//! let custom_signer = HWISigner::from_device(&first_device, Network::Testnet.into())?;
 //!
 //! # let mut wallet = Wallet::new_no_persist(
 //! #     "",
@@ -47,9 +47,9 @@
 //! # }
 //! ```
 
+use bitcoin::bip32::Fingerprint;
 use bitcoin::psbt::PartiallySignedTransaction;
 use bitcoin::secp256k1::{All, Secp256k1};
-use bitcoin::util::bip32::Fingerprint;
 
 use hwi::error::Error;
 use hwi::types::{HWIChain, HWIDevice};

crates/bdk/src/wallet/signer.rs

@@ -19,7 +19,7 @@
 //! # use core::str::FromStr;
 //! # use bitcoin::secp256k1::{Secp256k1, All};
 //! # use bitcoin::*;
-//! # use bitcoin::util::psbt;
+//! # use bitcoin::psbt;
 //! # use bdk::signer::*;
 //! # use bdk::*;
 //! # #[derive(Debug)]
@@ -76,34 +76,35 @@
 //!     Arc::new(custom_signer)
 //! );
 //!
-//! # Ok::<_, bdk::Error>(())
+//! # Ok::<_, anyhow::Error>(())
 //! ```
 
 use crate::collections::BTreeMap;
+use alloc::string::String;
 use alloc::sync::Arc;
 use alloc::vec::Vec;
 use core::cmp::Ordering;
 use core::fmt;
 use core::ops::{Bound::Included, Deref};
 
-use bitcoin::blockdata::opcodes;
-use bitcoin::blockdata::script::Builder as ScriptBuilder;
-use bitcoin::hashes::{hash160, Hash};
+use bitcoin::bip32::{ChildNumber, DerivationPath, ExtendedPrivKey, Fingerprint};
+use bitcoin::hashes::hash160;
 use bitcoin::secp256k1::Message;
-use bitcoin::util::bip32::{ChildNumber, DerivationPath, ExtendedPrivKey, Fingerprint};
-use bitcoin::util::{ecdsa, psbt, schnorr, sighash, taproot};
-use bitcoin::{secp256k1, XOnlyPublicKey};
-use bitcoin::{EcdsaSighashType, PrivateKey, PublicKey, SchnorrSighashType, Script};
+use bitcoin::sighash::{EcdsaSighashType, TapSighash, TapSighashType};
+use bitcoin::{ecdsa, psbt, sighash, taproot};
+use bitcoin::{key::TapTweak, key::XOnlyPublicKey, secp256k1};
+use bitcoin::{PrivateKey, PublicKey};
 
 use miniscript::descriptor::{
-    Descriptor, DescriptorPublicKey, DescriptorSecretKey, DescriptorXKey, KeyMap, SinglePriv,
-    SinglePubKey,
+    Descriptor, DescriptorMultiXKey, DescriptorPublicKey, DescriptorSecretKey, DescriptorXKey,
+    InnerXKey, KeyMap, SinglePriv, SinglePubKey,
 };
 use miniscript::{Legacy, Segwitv0, SigType, Tap, ToPublicKey};
 
 use super::utils::SecpCtx;
 use crate::descriptor::{DescriptorMeta, XKeyUtils};
 use crate::psbt::PsbtUtils;
+use crate::wallet::error::MiniscriptPsbtError;
 
 /// Identifier of a signer in the `SignersContainers`. Used as a key to find the right signer among
 /// multiple of them
@@ -130,7 +131,7 @@ impl From<Fingerprint> for SignerId {
 }
 
 /// Signing error
-#[derive(Debug, PartialEq, Eq, Clone)]
+#[derive(Debug)]
 pub enum SignerError {
     /// The private key is missing for the required public key
     MissingKey,
@@ -160,16 +161,12 @@ pub enum SignerError {
     InvalidSighash,
     /// Error while computing the hash to sign
     SighashError(sighash::Error),
-    /// Error while signing using hardware wallets
-    #[cfg(feature = "hardware-signer")]
-    HWIError(hwi::error::Error),
-}
-
-#[cfg(feature = "hardware-signer")]
-impl From<hwi::error::Error> for SignerError {
-    fn from(e: hwi::error::Error) -> Self {
-        SignerError::HWIError(e)
-    }
+    /// Miniscript PSBT error
+    MiniscriptPsbt(MiniscriptPsbtError),
+    /// To be used only by external libraries implementing [`InputSigner`] or
+    /// [`TransactionSigner`], so that they can return their own custom errors, without having to
+    /// modify [`SignerError`] in BDK.
+    External(String),
 }
 
 impl From<sighash::Error> for SignerError {
@@ -193,8 +190,8 @@ impl fmt::Display for SignerError {
             Self::NonStandardSighash => write!(f, "The psbt contains a non standard sighash"),
             Self::InvalidSighash => write!(f, "Invalid SIGHASH for the signing context in use"),
             Self::SighashError(err) => write!(f, "Error while computing the hash to sign: {}", err),
-            #[cfg(feature = "hardware-signer")]
-            Self::HWIError(err) => write!(f, "Error while signing using hardware wallets: {}", err),
+            Self::MiniscriptPsbt(err) => write!(f, "Miniscript PSBT error: {}", err),
+            Self::External(err) => write!(f, "{}", err),
         }
     }
 }
@@ -218,7 +215,7 @@ pub enum SignerContext {
     },
 }
 
-/// Wrapper structure to pair a signer with its context
+/// Wrapper to pair a signer with its context
 #[derive(Debug, Clone)]
 pub struct SignerWrapper<S: Sized + fmt::Debug + Clone> {
     signer: S,
@@ -383,6 +380,48 @@ impl InputSigner for SignerWrapper<DescriptorXKey<ExtendedPrivKey>> {
     }
 }
 
+fn multikey_to_xkeys<K: InnerXKey + Clone>(
+    multikey: DescriptorMultiXKey<K>,
+) -> Vec<DescriptorXKey<K>> {
+    multikey
+        .derivation_paths
+        .into_paths()
+        .into_iter()
+        .map(|derivation_path| DescriptorXKey {
+            origin: multikey.origin.clone(),
+            xkey: multikey.xkey.clone(),
+            derivation_path,
+            wildcard: multikey.wildcard,
+        })
+        .collect()
+}
+
+impl SignerCommon for SignerWrapper<DescriptorMultiXKey<ExtendedPrivKey>> {
+    fn id(&self, secp: &SecpCtx) -> SignerId {
+        SignerId::from(self.root_fingerprint(secp))
+    }
+
+    fn descriptor_secret_key(&self) -> Option<DescriptorSecretKey> {
+        Some(DescriptorSecretKey::MultiXPrv(self.signer.clone()))
+    }
+}
+
+impl InputSigner for SignerWrapper<DescriptorMultiXKey<ExtendedPrivKey>> {
+    fn sign_input(
+        &self,
+        psbt: &mut psbt::PartiallySignedTransaction,
+        input_index: usize,
+        sign_options: &SignOptions,
+        secp: &SecpCtx,
+    ) -> Result<(), SignerError> {
+        let xkeys = multikey_to_xkeys(self.signer.clone());
+        for xkey in xkeys {
+            SignerWrapper::new(xkey, self.ctx).sign_input(psbt, input_index, sign_options, secp)?
+        }
+        Ok(())
+    }
+}
+
 impl SignerCommon for SignerWrapper<PrivateKey> {
     fn id(&self, secp: &SecpCtx) -> SignerId {
         SignerId::from(self.public_key(secp).to_pubkeyhash(SigType::Ecdsa))
@@ -418,20 +457,23 @@ impl InputSigner for SignerWrapper<PrivateKey> {
         let x_only_pubkey = XOnlyPublicKey::from(pubkey.inner);
 
         if let SignerContext::Tap { is_internal_key } = self.ctx {
-            if is_internal_key
-                && psbt.inputs[input_index].tap_key_sig.is_none()
-                && sign_options.sign_with_tap_internal_key
-            {
-                let (hash, hash_ty) = Tap::sighash(psbt, input_index, None)?;
-                sign_psbt_schnorr(
-                    &self.inner,
-                    x_only_pubkey,
-                    None,
-                    &mut psbt.inputs[input_index],
-                    hash,
-                    hash_ty,
-                    secp,
-                );
+            if let Some(psbt_internal_key) = psbt.inputs[input_index].tap_internal_key {
+                if is_internal_key
+                    && psbt.inputs[input_index].tap_key_sig.is_none()
+                    && sign_options.sign_with_tap_internal_key
+                    && x_only_pubkey == psbt_internal_key
+                {
+                    let (hash, hash_ty) = Tap::sighash(psbt, input_index, None)?;
+                    sign_psbt_schnorr(
+                        &self.inner,
+                        x_only_pubkey,
+                        None,
+                        &mut psbt.inputs[input_index],
+                        hash,
+                        hash_ty,
+                        secp,
+                    );
+                }
             }
 
             if let Some((leaf_hashes, _)) =
@@ -477,8 +519,16 @@ impl InputSigner for SignerWrapper<PrivateKey> {
         }
 
         let (hash, hash_ty) = match self.ctx {
-            SignerContext::Segwitv0 => Segwitv0::sighash(psbt, input_index, ())?,
-            SignerContext::Legacy => Legacy::sighash(psbt, input_index, ())?,
+            SignerContext::Segwitv0 => {
+                let (h, t) = Segwitv0::sighash(psbt, input_index, ())?;
+                let h = h.to_raw_hash();
+                (h, t)
+            }
+            SignerContext::Legacy => {
+                let (h, t) = Legacy::sighash(psbt, input_index, ())?;
+                let h = h.to_raw_hash();
+                (h, t)
+            }
             _ => return Ok(()), // handled above
         };
         sign_psbt_ecdsa(
@@ -499,12 +549,12 @@ fn sign_psbt_ecdsa(
     secret_key: &secp256k1::SecretKey,
     pubkey: PublicKey,
     psbt_input: &mut psbt::Input,
-    hash: bitcoin::Sighash,
+    hash: impl bitcoin::hashes::Hash + bitcoin::secp256k1::ThirtyTwoByteHash,
     hash_ty: EcdsaSighashType,
     secp: &SecpCtx,
     allow_grinding: bool,
 ) {
-    let msg = &Message::from_slice(&hash.into_inner()[..]).unwrap();
+    let msg = &Message::from(hash);
     let sig = if allow_grinding {
         secp.sign_ecdsa_low_r(msg, secret_key)
     } else {
@@ -513,7 +563,7 @@ fn sign_psbt_ecdsa(
     secp.verify_ecdsa(msg, &sig, &pubkey.inner)
         .expect("invalid or corrupted ecdsa signature");
 
-    let final_signature = ecdsa::EcdsaSig { sig, hash_ty };
+    let final_signature = ecdsa::Signature { sig, hash_ty };
     psbt_input.partial_sigs.insert(pubkey, final_signature);
 }
 
@@ -523,12 +573,10 @@ fn sign_psbt_schnorr(
     pubkey: XOnlyPublicKey,
     leaf_hash: Option<taproot::TapLeafHash>,
     psbt_input: &mut psbt::Input,
-    hash: taproot::TapSighashHash,
-    hash_ty: SchnorrSighashType,
+    hash: TapSighash,
+    hash_ty: TapSighashType,
     secp: &SecpCtx,
 ) {
-    use schnorr::TapTweak;
-
     let keypair = secp256k1::KeyPair::from_seckey_slice(secp, secret_key.as_ref()).unwrap();
     let keypair = match leaf_hash {
         None => keypair
@@ -537,12 +585,12 @@ fn sign_psbt_schnorr(
         Some(_) => keypair, // no tweak for script spend
     };
 
-    let msg = &Message::from_slice(&hash.into_inner()[..]).unwrap();
+    let msg = &Message::from(hash);
     let sig = secp.sign_schnorr(msg, &keypair);
     secp.verify_schnorr(&sig, msg, &XOnlyPublicKey::from_keypair(&keypair).0)
         .expect("invalid or corrupted schnorr signature");
 
-    let final_signature = schnorr::SchnorrSig { sig, hash_ty };
+    let final_signature = taproot::Signature { sig, hash_ty };
 
     if let Some(lh) = leaf_hash {
         psbt_input
@@ -632,6 +680,11 @@ impl SignersContainer {
                     SignerOrdering::default(),
                     Arc::new(SignerWrapper::new(xprv, ctx)),
                 ),
+                DescriptorSecretKey::MultiXPrv(xprv) => container.add_external(
+                    SignerId::from(xprv.root_fingerprint(secp)),
+                    SignerOrdering::default(),
+                    Arc::new(SignerWrapper::new(xprv, ctx)),
+                ),
             };
         }
 
@@ -699,7 +752,7 @@ pub struct SignOptions {
     /// Whether the signer should trust the `witness_utxo`, if the `non_witness_utxo` hasn't been
     /// provided
     ///
-    /// Defaults to `false` to mitigate the "SegWit bug" which chould trick the wallet into
+    /// Defaults to `false` to mitigate the "SegWit bug" which should trick the wallet into
     /// paying a fee larger than expected.
     ///
     /// Some wallets, especially if relatively old, might not provide the `non_witness_utxo` for
@@ -753,9 +806,10 @@ pub struct SignOptions {
 }
 
 /// Customize which taproot script-path leaves the signer should sign.
-#[derive(Debug, Clone, PartialEq, Eq)]
+#[derive(Default, Debug, Clone, PartialEq, Eq)]
 pub enum TapLeavesOptions {
     /// The signer will sign all the leaves it has a key for.
+    #[default]
     All,
     /// The signer won't sign leaves other than the ones specified. Note that it could still ignore
     /// some of the specified leaves, if it doesn't have the right key to sign them.
@@ -766,13 +820,6 @@ pub enum TapLeavesOptions {
     None,
 }
 
-impl Default for TapLeavesOptions {
-    fn default() -> Self {
-        TapLeavesOptions::All
-    }
-}
-
-#[allow(clippy::derivable_impls)]
 impl Default for SignOptions {
     fn default() -> Self {
         SignOptions {
@@ -802,7 +849,7 @@ pub(crate) trait ComputeSighash {
 
 impl ComputeSighash for Legacy {
     type Extra = ();
-    type Sighash = bitcoin::Sighash;
+    type Sighash = sighash::LegacySighash;
     type SighashType = EcdsaSighashType;
 
     fn sighash(
@@ -849,19 +896,9 @@ impl ComputeSighash for Legacy {
     }
 }
 
-fn p2wpkh_script_code(script: &Script) -> Script {
-    ScriptBuilder::new()
-        .push_opcode(opcodes::all::OP_DUP)
-        .push_opcode(opcodes::all::OP_HASH160)
-        .push_slice(&script[2..])
-        .push_opcode(opcodes::all::OP_EQUALVERIFY)
-        .push_opcode(opcodes::all::OP_CHECKSIG)
-        .into_script()
-}
-
 impl ComputeSighash for Segwitv0 {
     type Extra = ();
-    type Sighash = bitcoin::Sighash;
+    type Sighash = sighash::SegwitV0Sighash;
     type SighashType = EcdsaSighashType;
 
     fn sighash(
@@ -908,14 +945,21 @@ impl ComputeSighash for Segwitv0 {
             Some(ref witness_script) => witness_script.clone(),
             None => {
                 if utxo.script_pubkey.is_v0_p2wpkh() {
-                    p2wpkh_script_code(&utxo.script_pubkey)
+                    utxo.script_pubkey
+                        .p2wpkh_script_code()
+                        .expect("We check above that the spk is a p2wpkh")
                 } else if psbt_input
                     .redeem_script
                     .as_ref()
-                    .map(Script::is_v0_p2wpkh)
+                    .map(|s| s.is_v0_p2wpkh())
                     .unwrap_or(false)
                 {
-                    p2wpkh_script_code(psbt_input.redeem_script.as_ref().unwrap())
+                    psbt_input
+                        .redeem_script
+                        .as_ref()
+                        .unwrap()
+                        .p2wpkh_script_code()
+                        .expect("We check above that the spk is a p2wpkh")
                 } else {
                     return Err(SignerError::MissingWitnessScript);
                 }
@@ -936,14 +980,14 @@ impl ComputeSighash for Segwitv0 {
 
 impl ComputeSighash for Tap {
     type Extra = Option<taproot::TapLeafHash>;
-    type Sighash = taproot::TapSighashHash;
-    type SighashType = SchnorrSighashType;
+    type Sighash = TapSighash;
+    type SighashType = TapSighashType;
 
     fn sighash(
         psbt: &psbt::PartiallySignedTransaction,
         input_index: usize,
         extra: Self::Extra,
-    ) -> Result<(Self::Sighash, SchnorrSighashType), SignerError> {
+    ) -> Result<(Self::Sighash, TapSighashType), SignerError> {
         if input_index >= psbt.inputs.len() || input_index >= psbt.unsigned_tx.input.len() {
             return Err(SignerError::InputIndexOutOfRange);
         }
@@ -952,8 +996,8 @@ impl ComputeSighash for Tap {
 
         let sighash_type = psbt_input
             .sighash_type
-            .unwrap_or_else(|| SchnorrSighashType::Default.into())
-            .schnorr_hash_ty()
+            .unwrap_or_else(|| TapSighashType::Default.into())
+            .taproot_hash_ty()
             .map_err(|_| SignerError::InvalidSighash)?;
         let witness_utxos = (0..psbt.inputs.len())
             .map(|i| psbt.get_utxo_for(i))
@@ -1015,8 +1059,8 @@ mod signers_container_tests {
     use crate::descriptor::IntoWalletDescriptor;
     use crate::keys::{DescriptorKey, IntoDescriptorKey};
     use assert_matches::assert_matches;
+    use bitcoin::bip32;
     use bitcoin::secp256k1::{All, Secp256k1};
-    use bitcoin::util::bip32;
     use bitcoin::Network;
     use core::str::FromStr;
     use miniscript::ScriptContext;

crates/bdk/src/wallet/tx_builder.rs

@@ -17,8 +17,12 @@
 //! # use std::str::FromStr;
 //! # use bitcoin::*;
 //! # use bdk::*;
+//! # use bdk::wallet::ChangeSet;
+//! # use bdk::wallet::error::CreateTxError;
 //! # use bdk::wallet::tx_builder::CreateTx;
-//! # let to_address = Address::from_str("2N4eQYCbKUHCCTUjBJeHcJp9ok6J2GZsTDt").unwrap();
+//! # use bdk_chain::PersistBackend;
+//! # use anyhow::Error;
+//! # let to_address = Address::from_str("2N4eQYCbKUHCCTUjBJeHcJp9ok6J2GZsTDt").unwrap().assume_checked();
 //! # let mut wallet = doctest_wallet!();
 //! // create a TxBuilder from a wallet
 //! let mut tx_builder = wallet.build_tx();
@@ -27,13 +31,13 @@
 //!     // Create a transaction with one output to `to_address` of 50_000 satoshi
 //!     .add_recipient(to_address.script_pubkey(), 50_000)
 //!     // With a custom fee rate of 5.0 satoshi/vbyte
-//!     .fee_rate(FeeRate::from_sat_per_vb(5.0))
+//!     .fee_rate(bdk::FeeRate::from_sat_per_vb(5.0))
 //!     // Only spend non-change outputs
 //!     .do_not_spend_change()
 //!     // Turn on RBF signaling
 //!     .enable_rbf();
-//! let (psbt, tx_details) = tx_builder.finish()?;
-//! # Ok::<(), bdk::Error>(())
+//! let psbt = tx_builder.finish()?;
+//! # Ok::<(), anyhow::Error>(())
 //! ```
 
 use crate::collections::BTreeMap;
@@ -41,18 +45,18 @@ use crate::collections::HashSet;
 use alloc::{boxed::Box, rc::Rc, string::String, vec::Vec};
 use bdk_chain::PersistBackend;
 use core::cell::RefCell;
+use core::fmt;
 use core::marker::PhantomData;
 
-use bitcoin::util::psbt::{self, PartiallySignedTransaction as Psbt};
-use bitcoin::{LockTime, OutPoint, Script, Sequence, Transaction};
+use bitcoin::psbt::{self, PartiallySignedTransaction as Psbt};
+use bitcoin::{absolute, script::PushBytes, OutPoint, ScriptBuf, Sequence, Transaction, Txid};
 
 use super::coin_selection::{CoinSelectionAlgorithm, DefaultCoinSelectionAlgorithm};
 use super::ChangeSet;
-use crate::{
-    types::{FeeRate, KeychainKind, LocalUtxo, WeightedUtxo},
-    TransactionDetails,
-};
-use crate::{Error, Utxo, Wallet};
+use crate::types::{FeeRate, KeychainKind, LocalOutput, WeightedUtxo};
+use crate::wallet::CreateTxError;
+use crate::{Utxo, Wallet};
+
 /// Context in which the [`TxBuilder`] is valid
 pub trait TxBuilderContext: core::fmt::Debug + Default + Clone {}
 
@@ -81,11 +85,15 @@ impl TxBuilderContext for BumpFee {}
 /// # use bdk::wallet::tx_builder::*;
 /// # use bitcoin::*;
 /// # use core::str::FromStr;
+/// # use bdk::wallet::ChangeSet;
+/// # use bdk::wallet::error::CreateTxError;
+/// # use bdk_chain::PersistBackend;
+/// # use anyhow::Error;
 /// # let mut wallet = doctest_wallet!();
-/// # let addr1 = Address::from_str("2N4eQYCbKUHCCTUjBJeHcJp9ok6J2GZsTDt").unwrap();
+/// # let addr1 = Address::from_str("2N4eQYCbKUHCCTUjBJeHcJp9ok6J2GZsTDt").unwrap().assume_checked();
 /// # let addr2 = addr1.clone();
 /// // chaining
-/// let (psbt1, details) = {
+/// let psbt1 = {
 ///     let mut builder = wallet.build_tx();
 ///     builder
 ///         .ordering(TxOrdering::Untouched)
@@ -95,7 +103,7 @@ impl TxBuilderContext for BumpFee {}
 /// };
 ///
 /// // non-chaining
-/// let (psbt2, details) = {
+/// let psbt2 = {
 ///     let mut builder = wallet.build_tx();
 ///     builder.ordering(TxOrdering::Untouched);
 ///     for addr in &[addr1, addr2] {
@@ -105,7 +113,7 @@ impl TxBuilderContext for BumpFee {}
 /// };
 ///
 /// assert_eq!(psbt1.unsigned_tx.output[..2], psbt2.unsigned_tx.output[..2]);
-/// # Ok::<(), bdk::Error>(())
+/// # Ok::<(), anyhow::Error>(())
 /// ```
 ///
 /// At the moment [`coin_selection`] is an exception to the rule as it consumes `self`.
@@ -129,9 +137,9 @@ pub struct TxBuilder<'a, D, Cs, Ctx> {
 //TODO: TxParams should eventually be exposed publicly.
 #[derive(Default, Debug, Clone)]
 pub(crate) struct TxParams {
-    pub(crate) recipients: Vec<(Script, u64)>,
+    pub(crate) recipients: Vec<(ScriptBuf, u64)>,
     pub(crate) drain_wallet: bool,
-    pub(crate) drain_to: Option<Script>,
+    pub(crate) drain_to: Option<ScriptBuf>,
     pub(crate) fee_policy: Option<FeePolicy>,
     pub(crate) internal_policy_path: Option<BTreeMap<String, Vec<usize>>>,
     pub(crate) external_policy_path: Option<BTreeMap<String, Vec<usize>>>,
@@ -140,7 +148,7 @@ pub(crate) struct TxParams {
     pub(crate) manually_selected_only: bool,
     pub(crate) sighash: Option<psbt::PsbtSighashType>,
     pub(crate) ordering: TxOrdering,
-    pub(crate) locktime: Option<LockTime>,
+    pub(crate) locktime: Option<absolute::LockTime>,
     pub(crate) rbf: Option<RbfValue>,
     pub(crate) version: Option<Version>,
     pub(crate) change_policy: ChangeSpendPolicy,
@@ -148,7 +156,7 @@ pub(crate) struct TxParams {
     pub(crate) add_global_xpubs: bool,
     pub(crate) include_output_redeem_witness_script: bool,
     pub(crate) bumping_fee: Option<PreviousFee>,
-    pub(crate) current_height: Option<LockTime>,
+    pub(crate) current_height: Option<absolute::LockTime>,
     pub(crate) allow_dust: bool,
 }
 
@@ -182,8 +190,19 @@ impl<'a, D, Cs: Clone, Ctx> Clone for TxBuilder<'a, D, Cs, Ctx> {
 }
 
 // methods supported by both contexts, for any CoinSelectionAlgorithm
-impl<'a, D, Cs: CoinSelectionAlgorithm, Ctx: TxBuilderContext> TxBuilder<'a, D, Cs, Ctx> {
+impl<'a, D, Cs, Ctx> TxBuilder<'a, D, Cs, Ctx> {
     /// Set a custom fee rate
+    /// The fee_rate method sets the mining fee paid by the transaction as a rate on its size.
+    /// This means that the total fee paid is equal to this rate * size of the transaction in virtual Bytes (vB) or Weight Unit (wu).
+    /// This rate is internally expressed in satoshis-per-virtual-bytes (sats/vB) using FeeRate::from_sat_per_vb, but can also be set by:
+    /// * sats/kvB (1000 sats/kvB == 1 sats/vB) using FeeRate::from_sat_per_kvb
+    /// * btc/kvB (0.00001000 btc/kvB == 1 sats/vB) using FeeRate::from_btc_per_kvb
+    /// * sats/kwu (250 sats/kwu == 1 sats/vB) using FeeRate::from_sat_per_kwu
+    /// Default is 1 sat/vB (see min_relay_fee)
+    ///
+    /// Note that this is really a minimum feerate -- it's possible to
+    /// overshoot it slightly since adding a change output to drain the remaining
+    /// excess might not be viable.
     pub fn fee_rate(&mut self, fee_rate: FeeRate) -> &mut Self {
         self.params.fee_policy = Some(FeePolicy::FeeRate(fee_rate));
         self
@@ -192,7 +211,12 @@ impl<'a, D, Cs: CoinSelectionAlgorithm, Ctx: TxBuilderContext> TxBuilder<'a, D,
     /// Set an absolute fee
     /// The fee_absolute method refers to the absolute transaction fee in satoshis (sats).
     /// If anyone sets both the fee_absolute method and the fee_rate method,
-    /// the fee_absolute value will take precedence over the fee_rate.
+    /// the FeePolicy enum will be set by whichever method was called last,
+    /// as the FeeRate and FeeAmount are mutually exclusive.
+    ///
+    /// Note that this is really a minimum absolute fee -- it's possible to
+    /// overshoot it slightly since adding a change output to drain the remaining
+    /// excess might not be viable.
     pub fn fee_absolute(&mut self, fee_amount: u64) -> &mut Self {
         self.params.fee_policy = Some(FeePolicy::FeeAmount(fee_amount));
         self
@@ -245,7 +269,10 @@ impl<'a, D, Cs: CoinSelectionAlgorithm, Ctx: TxBuilderContext> TxBuilder<'a, D,
     /// # use std::collections::BTreeMap;
     /// # use bitcoin::*;
     /// # use bdk::*;
-    /// # let to_address = Address::from_str("2N4eQYCbKUHCCTUjBJeHcJp9ok6J2GZsTDt").unwrap();
+    /// # let to_address =
+    /// Address::from_str("2N4eQYCbKUHCCTUjBJeHcJp9ok6J2GZsTDt")
+    ///     .unwrap()
+    ///     .assume_checked();
     /// # let mut wallet = doctest_wallet!();
     /// let mut path = BTreeMap::new();
     /// path.insert("aabbccdd".to_string(), vec![0, 1]);
@@ -255,7 +282,7 @@ impl<'a, D, Cs: CoinSelectionAlgorithm, Ctx: TxBuilderContext> TxBuilder<'a, D,
     ///     .add_recipient(to_address.script_pubkey(), 50_000)
     ///     .policy_path(path, KeychainKind::External);
     ///
-    /// # Ok::<(), bdk::Error>(())
+    /// # Ok::<(), anyhow::Error>(())
     /// ```
     pub fn policy_path(
         &mut self,
@@ -277,16 +304,21 @@ impl<'a, D, Cs: CoinSelectionAlgorithm, Ctx: TxBuilderContext> TxBuilder<'a, D,
     ///
     /// These have priority over the "unspendable" utxos, meaning that if a utxo is present both in
     /// the "utxos" and the "unspendable" list, it will be spent.
-    pub fn add_utxos(&mut self, outpoints: &[OutPoint]) -> Result<&mut Self, Error> {
+    pub fn add_utxos(&mut self, outpoints: &[OutPoint]) -> Result<&mut Self, AddUtxoError> {
         {
             let wallet = self.wallet.borrow();
             let utxos = outpoints
                 .iter()
-                .map(|outpoint| wallet.get_utxo(*outpoint).ok_or(Error::UnknownUtxo))
+                .map(|outpoint| {
+                    wallet
+                        .get_utxo(*outpoint)
+                        .ok_or(AddUtxoError::UnknownUtxo(*outpoint))
+                })
                 .collect::<Result<Vec<_>, _>>()?;
 
             for utxo in utxos {
                 let descriptor = wallet.get_descriptor_for_keychain(utxo.keychain);
+                #[allow(deprecated)]
                 let satisfaction_weight = descriptor.max_satisfaction_weight().unwrap();
                 self.params.utxos.push(WeightedUtxo {
                     satisfaction_weight,
@@ -302,7 +334,7 @@ impl<'a, D, Cs: CoinSelectionAlgorithm, Ctx: TxBuilderContext> TxBuilder<'a, D,
     ///
     /// These have priority over the "unspendable" utxos, meaning that if a utxo is present both in
     /// the "utxos" and the "unspendable" list, it will be spent.
-    pub fn add_utxo(&mut self, outpoint: OutPoint) -> Result<&mut Self, Error> {
+    pub fn add_utxo(&mut self, outpoint: OutPoint) -> Result<&mut Self, AddUtxoError> {
         self.add_utxos(&[outpoint])
     }
 
@@ -334,6 +366,10 @@ impl<'a, D, Cs: CoinSelectionAlgorithm, Ctx: TxBuilderContext> TxBuilder<'a, D,
     ///
     /// This is an **EXPERIMENTAL** feature, API and other major changes are expected.
     ///
+    /// In order to use [`Wallet::calculate_fee`] or [`Wallet::calculate_fee_rate`] for a transaction
+    /// created with foreign UTXO(s) you must manually insert the corresponding TxOut(s) into the tx
+    /// graph using the [`Wallet::insert_txout`] function.
+    ///
     /// # Errors
     ///
     /// This method returns errors in the following circumstances:
@@ -353,23 +389,38 @@ impl<'a, D, Cs: CoinSelectionAlgorithm, Ctx: TxBuilderContext> TxBuilder<'a, D,
         outpoint: OutPoint,
         psbt_input: psbt::Input,
         satisfaction_weight: usize,
-    ) -> Result<&mut Self, Error> {
+    ) -> Result<&mut Self, AddForeignUtxoError> {
+        self.add_foreign_utxo_with_sequence(
+            outpoint,
+            psbt_input,
+            satisfaction_weight,
+            Sequence::MAX,
+        )
+    }
+
+    /// Same as [add_foreign_utxo](TxBuilder::add_foreign_utxo) but allows to set the nSequence value.
+    pub fn add_foreign_utxo_with_sequence(
+        &mut self,
+        outpoint: OutPoint,
+        psbt_input: psbt::Input,
+        satisfaction_weight: usize,
+        sequence: Sequence,
+    ) -> Result<&mut Self, AddForeignUtxoError> {
         if psbt_input.witness_utxo.is_none() {
             match psbt_input.non_witness_utxo.as_ref() {
                 Some(tx) => {
                     if tx.txid() != outpoint.txid {
-                        return Err(Error::Generic(
-                            "Foreign utxo outpoint does not match PSBT input".into(),
-                        ));
+                        return Err(AddForeignUtxoError::InvalidTxid {
+                            input_txid: tx.txid(),
+                            foreign_utxo: outpoint,
+                        });
                     }
                     if tx.output.len() <= outpoint.vout as usize {
-                        return Err(Error::InvalidOutpoint(outpoint));
+                        return Err(AddForeignUtxoError::InvalidOutpoint(outpoint));
                     }
                 }
                 None => {
-                    return Err(Error::Generic(
-                        "Foreign utxo missing witness_utxo or non_witness_utxo".into(),
-                    ))
+                    return Err(AddForeignUtxoError::MissingUtxo);
                 }
             }
         }
@@ -378,6 +429,7 @@ impl<'a, D, Cs: CoinSelectionAlgorithm, Ctx: TxBuilderContext> TxBuilder<'a, D,
             satisfaction_weight,
             utxo: Utxo::Foreign {
                 outpoint,
+                sequence: Some(sequence),
                 psbt_input: Box::new(psbt_input),
             },
         });
@@ -431,7 +483,7 @@ impl<'a, D, Cs: CoinSelectionAlgorithm, Ctx: TxBuilderContext> TxBuilder<'a, D,
     /// Use a specific nLockTime while creating the transaction
     ///
     /// This can cause conflicts if the wallet's descriptors contain an "after" (OP_CLTV) operator.
-    pub fn nlocktime(&mut self, locktime: LockTime) -> &mut Self {
+    pub fn nlocktime(&mut self, locktime: absolute::LockTime) -> &mut Self {
         self.params.locktime = Some(locktime);
         self
     }
@@ -470,7 +522,7 @@ impl<'a, D, Cs: CoinSelectionAlgorithm, Ctx: TxBuilderContext> TxBuilder<'a, D,
         self
     }
 
-    /// Only Fill-in the [`psbt::Input::witness_utxo`](bitcoin::util::psbt::Input::witness_utxo) field when spending from
+    /// Only Fill-in the [`psbt::Input::witness_utxo`](bitcoin::psbt::Input::witness_utxo) field when spending from
     /// SegWit descriptors.
     ///
     /// This reduces the size of the PSBT, but some signers might reject them due to the lack of
@@ -480,8 +532,8 @@ impl<'a, D, Cs: CoinSelectionAlgorithm, Ctx: TxBuilderContext> TxBuilder<'a, D,
         self
     }
 
-    /// Fill-in the [`psbt::Output::redeem_script`](bitcoin::util::psbt::Output::redeem_script) and
-    /// [`psbt::Output::witness_script`](bitcoin::util::psbt::Output::witness_script) fields.
+    /// Fill-in the [`psbt::Output::redeem_script`](bitcoin::psbt::Output::redeem_script) and
+    /// [`psbt::Output::witness_script`](bitcoin::psbt::Output::witness_script) fields.
     ///
     /// This is useful for signers which always require it, like ColdCard hardware wallets.
     pub fn include_output_redeem_witness_script(&mut self) -> &mut Self {
@@ -507,7 +559,7 @@ impl<'a, D, Cs: CoinSelectionAlgorithm, Ctx: TxBuilderContext> TxBuilder<'a, D,
 
     /// Choose the coin selection algorithm
     ///
-    /// Overrides the [`DefaultCoinSelectionAlgorithm`](super::coin_selection::DefaultCoinSelectionAlgorithm).
+    /// Overrides the [`DefaultCoinSelectionAlgorithm`].
     ///
     /// Note that this function consumes the builder and returns it so it is usually best to put this as the first call on the builder.
     pub fn coin_selection<P: CoinSelectionAlgorithm>(
@@ -522,20 +574,6 @@ impl<'a, D, Cs: CoinSelectionAlgorithm, Ctx: TxBuilderContext> TxBuilder<'a, D,
         }
     }
 
-    /// Finish building the transaction.
-    ///
-    /// Returns the [`BIP174`] "PSBT" and summary details about the transaction.
-    ///
-    /// [`BIP174`]: https://github.com/bitcoin/bips/blob/master/bip-0174.mediawiki
-    pub fn finish(self) -> Result<(Psbt, TransactionDetails), Error>
-    where
-        D: PersistBackend<ChangeSet>,
-    {
-        self.wallet
-            .borrow_mut()
-            .create_tx(self.coin_selection, self.params)
-    }
-
     /// Enable signaling RBF
     ///
     /// This will use the default nSequence value of `0xFFFFFFFD`.
@@ -568,7 +606,8 @@ impl<'a, D, Cs: CoinSelectionAlgorithm, Ctx: TxBuilderContext> TxBuilder<'a, D,
     ///
     /// In both cases, if you don't provide a current height, we use the last sync height.
     pub fn current_height(&mut self, height: u32) -> &mut Self {
-        self.params.current_height = Some(LockTime::from_height(height).expect("Invalid height"));
+        self.params.current_height =
+            Some(absolute::LockTime::from_height(height).expect("Invalid height"));
         self
     }
 
@@ -581,22 +620,122 @@ impl<'a, D, Cs: CoinSelectionAlgorithm, Ctx: TxBuilderContext> TxBuilder<'a, D,
     }
 }
 
+impl<'a, D, Cs: CoinSelectionAlgorithm, Ctx> TxBuilder<'a, D, Cs, Ctx> {
+    /// Finish building the transaction.
+    ///
+    /// Returns a new [`Psbt`] per [`BIP174`].
+    ///
+    /// [`BIP174`]: https://github.com/bitcoin/bips/blob/master/bip-0174.mediawiki
+    pub fn finish(self) -> Result<Psbt, CreateTxError<D::WriteError>>
+    where
+        D: PersistBackend<ChangeSet>,
+    {
+        self.wallet
+            .borrow_mut()
+            .create_tx(self.coin_selection, self.params)
+    }
+}
+
+#[derive(Debug)]
+/// Error returned from [`TxBuilder::add_utxo`] and [`TxBuilder::add_utxos`]
+pub enum AddUtxoError {
+    /// Happens when trying to spend an UTXO that is not in the internal database
+    UnknownUtxo(OutPoint),
+}
+
+impl fmt::Display for AddUtxoError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Self::UnknownUtxo(outpoint) => write!(
+                f,
+                "UTXO not found in the internal database for txid: {} with vout: {}",
+                outpoint.txid, outpoint.vout
+            ),
+        }
+    }
+}
+
+#[cfg(feature = "std")]
+impl std::error::Error for AddUtxoError {}
+
+#[derive(Debug)]
+/// Error returned from [`TxBuilder::add_foreign_utxo`].
+pub enum AddForeignUtxoError {
+    /// Foreign utxo outpoint txid does not match PSBT input txid
+    InvalidTxid {
+        /// PSBT input txid
+        input_txid: Txid,
+        /// Foreign UTXO outpoint
+        foreign_utxo: OutPoint,
+    },
+    /// Requested outpoint doesn't exist in the tx (vout greater than available outputs)
+    InvalidOutpoint(OutPoint),
+    /// Foreign utxo missing witness_utxo or non_witness_utxo
+    MissingUtxo,
+}
+
+impl fmt::Display for AddForeignUtxoError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Self::InvalidTxid {
+                input_txid,
+                foreign_utxo,
+            } => write!(
+                f,
+                "Foreign UTXO outpoint txid: {} does not match PSBT input txid: {}",
+                foreign_utxo.txid, input_txid,
+            ),
+            Self::InvalidOutpoint(outpoint) => write!(
+                f,
+                "Requested outpoint doesn't exist for txid: {} with vout: {}",
+                outpoint.txid, outpoint.vout,
+            ),
+            Self::MissingUtxo => write!(f, "Foreign utxo missing witness_utxo or non_witness_utxo"),
+        }
+    }
+}
+
+#[cfg(feature = "std")]
+impl std::error::Error for AddForeignUtxoError {}
+
+#[derive(Debug)]
+/// Error returned from [`TxBuilder::allow_shrinking`]
+pub enum AllowShrinkingError {
+    /// Script/PubKey was not in the original transaction
+    MissingScriptPubKey(ScriptBuf),
+}
+
+impl fmt::Display for AllowShrinkingError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Self::MissingScriptPubKey(script_buf) => write!(
+                f,
+                "Script/PubKey was not in the original transaction: {}",
+                script_buf,
+            ),
+        }
+    }
+}
+
+#[cfg(feature = "std")]
+impl std::error::Error for AllowShrinkingError {}
+
 impl<'a, D, Cs: CoinSelectionAlgorithm> TxBuilder<'a, D, Cs, CreateTx> {
     /// Replace the recipients already added with a new list
-    pub fn set_recipients(&mut self, recipients: Vec<(Script, u64)>) -> &mut Self {
+    pub fn set_recipients(&mut self, recipients: Vec<(ScriptBuf, u64)>) -> &mut Self {
         self.params.recipients = recipients;
         self
     }
 
     /// Add a recipient to the internal list
-    pub fn add_recipient(&mut self, script_pubkey: Script, amount: u64) -> &mut Self {
+    pub fn add_recipient(&mut self, script_pubkey: ScriptBuf, amount: u64) -> &mut Self {
         self.params.recipients.push((script_pubkey, amount));
         self
     }
 
     /// Add data as an output, using OP_RETURN
-    pub fn add_data(&mut self, data: &[u8]) -> &mut Self {
-        let script = Script::new_op_return(data);
+    pub fn add_data<T: AsRef<PushBytes>>(&mut self, data: &T) -> &mut Self {
+        let script = ScriptBuf::new_op_return(data);
         self.add_recipient(script, 0u64);
         self
     }
@@ -625,8 +764,15 @@ impl<'a, D, Cs: CoinSelectionAlgorithm> TxBuilder<'a, D, Cs, CreateTx> {
     /// # use std::str::FromStr;
     /// # use bitcoin::*;
     /// # use bdk::*;
+    /// # use bdk::wallet::ChangeSet;
+    /// # use bdk::wallet::error::CreateTxError;
     /// # use bdk::wallet::tx_builder::CreateTx;
-    /// # let to_address = Address::from_str("2N4eQYCbKUHCCTUjBJeHcJp9ok6J2GZsTDt").unwrap();
+    /// # use bdk_chain::PersistBackend;
+    /// # use anyhow::Error;
+    /// # let to_address =
+    /// Address::from_str("2N4eQYCbKUHCCTUjBJeHcJp9ok6J2GZsTDt")
+    ///     .unwrap()
+    ///     .assume_checked();
     /// # let mut wallet = doctest_wallet!();
     /// let mut tx_builder = wallet.build_tx();
     ///
@@ -635,17 +781,17 @@ impl<'a, D, Cs: CoinSelectionAlgorithm> TxBuilder<'a, D, Cs, CreateTx> {
     ///     .drain_wallet()
     ///     // Send the excess (which is all the coins minus the fee) to this address.
     ///     .drain_to(to_address.script_pubkey())
-    ///     .fee_rate(FeeRate::from_sat_per_vb(5.0))
+    ///     .fee_rate(bdk::FeeRate::from_sat_per_vb(5.0))
     ///     .enable_rbf();
-    /// let (psbt, tx_details) = tx_builder.finish()?;
-    /// # Ok::<(), bdk::Error>(())
+    /// let psbt = tx_builder.finish()?;
+    /// # Ok::<(), anyhow::Error>(())
     /// ```
     ///
     /// [`allow_shrinking`]: Self::allow_shrinking
     /// [`add_recipient`]: Self::add_recipient
     /// [`add_utxos`]: Self::add_utxos
     /// [`drain_wallet`]: Self::drain_wallet
-    pub fn drain_to(&mut self, script_pubkey: Script) -> &mut Self {
+    pub fn drain_to(&mut self, script_pubkey: ScriptBuf) -> &mut Self {
         self.params.drain_to = Some(script_pubkey);
         self
     }
@@ -663,7 +809,10 @@ impl<'a, D> TxBuilder<'a, D, DefaultCoinSelectionAlgorithm, BumpFee> {
     ///
     /// Returns an `Err` if `script_pubkey` can't be found among the recipients of the
     /// transaction we are bumping.
-    pub fn allow_shrinking(&mut self, script_pubkey: Script) -> Result<&mut Self, Error> {
+    pub fn allow_shrinking(
+        &mut self,
+        script_pubkey: ScriptBuf,
+    ) -> Result<&mut Self, AllowShrinkingError> {
         match self
             .params
             .recipients
@@ -675,18 +824,16 @@ impl<'a, D> TxBuilder<'a, D, DefaultCoinSelectionAlgorithm, BumpFee> {
                 self.params.drain_to = Some(script_pubkey);
                 Ok(self)
             }
-            None => Err(Error::Generic(format!(
-                "{} was not in the original transaction",
-                script_pubkey
-            ))),
+            None => Err(AllowShrinkingError::MissingScriptPubKey(script_pubkey)),
         }
     }
 }
 
 /// Ordering of the transaction's inputs and outputs
-#[derive(Debug, Ord, PartialOrd, Eq, PartialEq, Hash, Clone, Copy)]
+#[derive(Default, Debug, Ord, PartialOrd, Eq, PartialEq, Hash, Clone, Copy)]
 pub enum TxOrdering {
     /// Randomized (default)
+    #[default]
     Shuffle,
     /// Unchanged
     Untouched,
@@ -694,12 +841,6 @@ pub enum TxOrdering {
     Bip69Lexicographic,
 }
 
-impl Default for TxOrdering {
-    fn default() -> Self {
-        TxOrdering::Shuffle
-    }
-}
-
 impl TxOrdering {
     /// Sort transaction inputs and outputs by [`TxOrdering`] variant
     pub fn sort_tx(&self, tx: &mut Transaction) {
@@ -753,9 +894,10 @@ impl RbfValue {
 }
 
 /// Policy regarding the use of change outputs when creating a transaction
-#[derive(Debug, Ord, PartialOrd, Eq, PartialEq, Hash, Clone, Copy)]
+#[derive(Default, Debug, Ord, PartialOrd, Eq, PartialEq, Hash, Clone, Copy)]
 pub enum ChangeSpendPolicy {
     /// Use both change and non-change outputs (default)
+    #[default]
     ChangeAllowed,
     /// Only use change outputs (see [`TxBuilder::only_spend_change`])
     OnlyChange,
@@ -763,14 +905,8 @@ pub enum ChangeSpendPolicy {
     ChangeForbidden,
 }
 
-impl Default for ChangeSpendPolicy {
-    fn default() -> Self {
-        ChangeSpendPolicy::ChangeAllowed
-    }
-}
-
 impl ChangeSpendPolicy {
-    pub(crate) fn is_satisfied_by(&self, utxo: &LocalUtxo) -> bool {
+    pub(crate) fn is_satisfied_by(&self, utxo: &LocalOutput) -> bool {
         match self {
             ChangeSpendPolicy::ChangeAllowed => true,
             ChangeSpendPolicy::OnlyChange => utxo.keychain == KeychainKind::Internal,
@@ -868,17 +1004,20 @@ mod test {
         );
 
         assert_eq!(tx.output[0].value, 800);
-        assert_eq!(tx.output[1].script_pubkey, From::from(vec![0xAA]));
-        assert_eq!(tx.output[2].script_pubkey, From::from(vec![0xAA, 0xEE]));
+        assert_eq!(tx.output[1].script_pubkey, ScriptBuf::from(vec![0xAA]));
+        assert_eq!(
+            tx.output[2].script_pubkey,
+            ScriptBuf::from(vec![0xAA, 0xEE])
+        );
     }
 
-    fn get_test_utxos() -> Vec<LocalUtxo> {
+    fn get_test_utxos() -> Vec<LocalOutput> {
         use bitcoin::hashes::Hash;
 
         vec![
-            LocalUtxo {
+            LocalOutput {
                 outpoint: OutPoint {
-                    txid: bitcoin::Txid::from_inner([0; 32]),
+                    txid: bitcoin::Txid::from_slice(&[0; 32]).unwrap(),
                     vout: 0,
                 },
                 txout: Default::default(),
@@ -887,9 +1026,9 @@ mod test {
                 confirmation_time: ConfirmationTime::Unconfirmed { last_seen: 0 },
                 derivation_index: 0,
             },
-            LocalUtxo {
+            LocalOutput {
                 outpoint: OutPoint {
-                    txid: bitcoin::Txid::from_inner([0; 32]),
+                    txid: bitcoin::Txid::from_slice(&[0; 32]).unwrap(),
                     vout: 1,
                 },
                 txout: Default::default(),

crates/bdk/src/wallet/utils.rs

@@ -10,7 +10,7 @@
 // licenses.
 
 use bitcoin::secp256k1::{All, Secp256k1};
-use bitcoin::{LockTime, Script, Sequence};
+use bitcoin::{absolute, Script, Sequence};
 
 use miniscript::{MiniscriptKey, Satisfier, ToPublicKey};
 
@@ -65,7 +65,7 @@ pub(crate) fn check_nsequence_rbf(rbf: Sequence, csv: Sequence) -> bool {
 }
 
 impl<Pk: MiniscriptKey + ToPublicKey> Satisfier<Pk> for After {
-    fn check_after(&self, n: LockTime) -> bool {
+    fn check_after(&self, n: absolute::LockTime) -> bool {
         if let Some(current_height) = self.current_height {
             current_height >= n.to_consensus_u32()
         } else {
@@ -119,12 +119,14 @@ mod test {
     pub(crate) const SEQUENCE_LOCKTIME_TYPE_FLAG: u32 = 1 << 22;
 
     use super::{check_nsequence_rbf, IsDust};
-    use crate::bitcoin::{Address, Sequence};
+    use crate::bitcoin::{Address, Network, Sequence};
     use core::str::FromStr;
 
     #[test]
     fn test_is_dust() {
         let script_p2pkh = Address::from_str("1GNgwA8JfG7Kc8akJ8opdNWJUihqUztfPe")
+            .unwrap()
+            .require_network(Network::Bitcoin)
             .unwrap()
             .script_pubkey();
         assert!(script_p2pkh.is_p2pkh());
@@ -132,6 +134,8 @@ mod test {
         assert!(!546.is_dust(&script_p2pkh));
 
         let script_p2wpkh = Address::from_str("bc1qxlh2mnc0yqwas76gqq665qkggee5m98t8yskd8")
+            .unwrap()
+            .require_network(Network::Bitcoin)
             .unwrap()
             .script_pubkey();
         assert!(script_p2wpkh.is_v0_p2wpkh());

crates/bdk/tests/common.rs

@@ -1,25 +1,68 @@
 #![allow(unused)]
-use bdk::{wallet::AddressIndex, Wallet};
+
+use bdk::{wallet::AddressIndex, KeychainKind, LocalOutput, Wallet};
+use bdk_chain::indexed_tx_graph::Indexer;
 use bdk_chain::{BlockId, ConfirmationTime};
 use bitcoin::hashes::Hash;
-use bitcoin::{BlockHash, Network, Transaction, TxOut};
+use bitcoin::{Address, BlockHash, Network, OutPoint, Transaction, TxIn, TxOut, Txid};
+use std::str::FromStr;
 
-/// Return a fake wallet that appears to be funded for testing.
+// Return a fake wallet that appears to be funded for testing.
+//
+// The funded wallet containing a tx with a 76_000 sats input and two outputs, one spending 25_000
+// to a foreign address and one returning 50_000 back to the wallet as change. The remaining 1000
+// sats are the transaction fee.
 pub fn get_funded_wallet_with_change(
     descriptor: &str,
     change: Option<&str>,
 ) -> (Wallet, bitcoin::Txid) {
     let mut wallet = Wallet::new_no_persist(descriptor, change, Network::Regtest).unwrap();
-    let address = wallet.get_address(AddressIndex::New).address;
+    let change_address = wallet.get_address(AddressIndex::New).address;
+    let sendto_address = Address::from_str("bcrt1q3qtze4ys45tgdvguj66zrk4fu6hq3a3v9pfly5")
+        .expect("address")
+        .require_network(Network::Regtest)
+        .unwrap();
 
-    let tx = Transaction {
+    let tx0 = Transaction {
         version: 1,
-        lock_time: bitcoin::PackedLockTime(0),
-        input: vec![],
-        output: vec![TxOut {
-            value: 50_000,
-            script_pubkey: address.script_pubkey(),
+        lock_time: bitcoin::absolute::LockTime::ZERO,
+        input: vec![TxIn {
+            previous_output: OutPoint {
+                txid: Txid::all_zeros(),
+                vout: 0,
+            },
+            script_sig: Default::default(),
+            sequence: Default::default(),
+            witness: Default::default(),
         }],
+        output: vec![TxOut {
+            value: 76_000,
+            script_pubkey: change_address.script_pubkey(),
+        }],
+    };
+
+    let tx1 = Transaction {
+        version: 1,
+        lock_time: bitcoin::absolute::LockTime::ZERO,
+        input: vec![TxIn {
+            previous_output: OutPoint {
+                txid: tx0.txid(),
+                vout: 0,
+            },
+            script_sig: Default::default(),
+            sequence: Default::default(),
+            witness: Default::default(),
+        }],
+        output: vec![
+            TxOut {
+                value: 50_000,
+                script_pubkey: change_address.script_pubkey(),
+            },
+            TxOut {
+                value: 25_000,
+                script_pubkey: sendto_address.script_pubkey(),
+            },
+        ],
     };
 
     wallet
@@ -28,19 +71,39 @@ pub fn get_funded_wallet_with_change(
             hash: BlockHash::all_zeros(),
         })
         .unwrap();
+    wallet
+        .insert_checkpoint(BlockId {
+            height: 2_000,
+            hash: BlockHash::all_zeros(),
+        })
+        .unwrap();
     wallet
         .insert_tx(
-            tx.clone(),
+            tx0,
             ConfirmationTime::Confirmed {
                 height: 1_000,
                 time: 100,
             },
         )
         .unwrap();
+    wallet
+        .insert_tx(
+            tx1.clone(),
+            ConfirmationTime::Confirmed {
+                height: 2_000,
+                time: 200,
+            },
+        )
+        .unwrap();
 
-    (wallet, tx.txid())
+    (wallet, tx1.txid())
 }
 
+// Return a fake wallet that appears to be funded for testing.
+//
+// The funded wallet containing a tx with a 76_000 sats input and two outputs, one spending 25_000
+// to a foreign address and one returning 50_000 back to the wallet as change. The remaining 1000
+// sats are the transaction fee.
 pub fn get_funded_wallet(descriptor: &str) -> (Wallet, bitcoin::Txid) {
     get_funded_wallet_with_change(descriptor, None)
 }

crates/bdk/tests/psbt.rs

@@ -2,7 +2,7 @@ use bdk::bitcoin::TxIn;
 use bdk::wallet::AddressIndex;
 use bdk::wallet::AddressIndex::New;
 use bdk::{psbt, FeeRate, SignOptions};
-use bitcoin::util::psbt::PartiallySignedTransaction as Psbt;
+use bitcoin::psbt::PartiallySignedTransaction as Psbt;
 use core::str::FromStr;
 mod common;
 use common::*;
@@ -18,7 +18,7 @@ fn test_psbt_malformed_psbt_input_legacy() {
     let send_to = wallet.get_address(AddressIndex::New);
     let mut builder = wallet.build_tx();
     builder.add_recipient(send_to.script_pubkey(), 10_000);
-    let (mut psbt, _) = builder.finish().unwrap();
+    let mut psbt = builder.finish().unwrap();
     psbt.inputs.push(psbt_bip.inputs[0].clone());
     let options = SignOptions {
         trust_witness_utxo: true,
@@ -35,7 +35,7 @@ fn test_psbt_malformed_psbt_input_segwit() {
     let send_to = wallet.get_address(AddressIndex::New);
     let mut builder = wallet.build_tx();
     builder.add_recipient(send_to.script_pubkey(), 10_000);
-    let (mut psbt, _) = builder.finish().unwrap();
+    let mut psbt = builder.finish().unwrap();
     psbt.inputs.push(psbt_bip.inputs[1].clone());
     let options = SignOptions {
         trust_witness_utxo: true,
@@ -51,7 +51,7 @@ fn test_psbt_malformed_tx_input() {
     let send_to = wallet.get_address(AddressIndex::New);
     let mut builder = wallet.build_tx();
     builder.add_recipient(send_to.script_pubkey(), 10_000);
-    let (mut psbt, _) = builder.finish().unwrap();
+    let mut psbt = builder.finish().unwrap();
     psbt.unsigned_tx.input.push(TxIn::default());
     let options = SignOptions {
         trust_witness_utxo: true,
@@ -67,7 +67,7 @@ fn test_psbt_sign_with_finalized() {
     let send_to = wallet.get_address(AddressIndex::New);
     let mut builder = wallet.build_tx();
     builder.add_recipient(send_to.script_pubkey(), 10_000);
-    let (mut psbt, _) = builder.finish().unwrap();
+    let mut psbt = builder.finish().unwrap();
 
     // add a finalized input
     psbt.inputs.push(psbt_bip.inputs[0].clone());
@@ -89,7 +89,7 @@ fn test_psbt_fee_rate_with_witness_utxo() {
     let mut builder = wallet.build_tx();
     builder.drain_to(addr.script_pubkey()).drain_wallet();
     builder.fee_rate(FeeRate::from_sat_per_vb(expected_fee_rate));
-    let (mut psbt, _) = builder.finish().unwrap();
+    let mut psbt = builder.finish().unwrap();
     let fee_amount = psbt.fee_amount();
     assert!(fee_amount.is_some());
 
@@ -114,7 +114,7 @@ fn test_psbt_fee_rate_with_nonwitness_utxo() {
     let mut builder = wallet.build_tx();
     builder.drain_to(addr.script_pubkey()).drain_wallet();
     builder.fee_rate(FeeRate::from_sat_per_vb(expected_fee_rate));
-    let (mut psbt, _) = builder.finish().unwrap();
+    let mut psbt = builder.finish().unwrap();
     let fee_amount = psbt.fee_amount();
     assert!(fee_amount.is_some());
     let unfinalized_fee_rate = psbt.fee_rate().unwrap();
@@ -138,7 +138,7 @@ fn test_psbt_fee_rate_with_missing_txout() {
     let mut builder = wpkh_wallet.build_tx();
     builder.drain_to(addr.script_pubkey()).drain_wallet();
     builder.fee_rate(FeeRate::from_sat_per_vb(expected_fee_rate));
-    let (mut wpkh_psbt, _) = builder.finish().unwrap();
+    let mut wpkh_psbt = builder.finish().unwrap();
 
     wpkh_psbt.inputs[0].witness_utxo = None;
     wpkh_psbt.inputs[0].non_witness_utxo = None;
@@ -150,9 +150,43 @@ fn test_psbt_fee_rate_with_missing_txout() {
     let mut builder = pkh_wallet.build_tx();
     builder.drain_to(addr.script_pubkey()).drain_wallet();
     builder.fee_rate(FeeRate::from_sat_per_vb(expected_fee_rate));
-    let (mut pkh_psbt, _) = builder.finish().unwrap();
+    let mut pkh_psbt = builder.finish().unwrap();
 
     pkh_psbt.inputs[0].non_witness_utxo = None;
     assert!(pkh_psbt.fee_amount().is_none());
     assert!(pkh_psbt.fee_rate().is_none());
 }
+
+#[test]
+fn test_psbt_multiple_internalkey_signers() {
+    use bdk::signer::{SignerContext, SignerOrdering, SignerWrapper};
+    use bdk::KeychainKind;
+    use bitcoin::{secp256k1::Secp256k1, PrivateKey};
+    use miniscript::psbt::PsbtExt;
+    use std::sync::Arc;
+
+    let secp = Secp256k1::new();
+    let (mut wallet, _) = get_funded_wallet(get_test_tr_single_sig());
+    let send_to = wallet.get_address(AddressIndex::New);
+    let mut builder = wallet.build_tx();
+    builder.add_recipient(send_to.script_pubkey(), 10_000);
+    let mut psbt = builder.finish().unwrap();
+    // Adds a signer for the wrong internal key, bdk should not use this key to sign
+    wallet.add_signer(
+        KeychainKind::External,
+        // A signerordering lower than 100, bdk will use this signer first
+        SignerOrdering(0),
+        Arc::new(SignerWrapper::new(
+            PrivateKey::from_wif("5J5PZqvCe1uThJ3FZeUUFLCh2FuK9pZhtEK4MzhNmugqTmxCdwE").unwrap(),
+            SignerContext::Tap {
+                is_internal_key: true,
+            },
+        )),
+    );
+    let _ = wallet.sign(&mut psbt, SignOptions::default()).unwrap();
+    // Checks that we signed using the right key
+    assert!(
+        psbt.finalize_mut(&secp).is_ok(),
+        "The wrong internal key was used"
+    );
+}

crates/bitcoind_rpc/Cargo.toml

@@ -0,0 +1,28 @@
+[package]
+name = "bdk_bitcoind_rpc"
+version = "0.6.0"
+edition = "2021"
+rust-version = "1.63"
+homepage = "https://bitcoindevkit.org"
+repository = "https://github.com/bitcoindevkit/bdk"
+documentation = "https://docs.rs/bdk_bitcoind_rpc"
+description = "This crate is used for emitting blockchain data from the `bitcoind` RPC interface."
+license = "MIT OR Apache-2.0"
+readme = "README.md"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+# For no-std, remember to enable the bitcoin/no-std feature
+bitcoin = { version = "0.30", default-features = false }
+bitcoincore-rpc = { version = "0.17" }
+bdk_chain = { path = "../chain", version = "0.11", default-features = false }
+
+[dev-dependencies]
+bitcoind = { version = "0.33", features = ["25_0"] }
+anyhow = { version = "1" }
+
+[features]
+default = ["std"]
+std = ["bitcoin/std", "bdk_chain/std"]
+serde = ["bitcoin/serde", "bdk_chain/serde"]

crates/bitcoind_rpc/README.md

@@ -0,0 +1,3 @@
+# BDK Bitcoind RPC
+
+This crate is used for emitting blockchain data from the `bitcoind` RPC interface.

crates/bitcoind_rpc/src/lib.rs

@@ -0,0 +1,328 @@
+//! This crate is used for emitting blockchain data from the `bitcoind` RPC interface. It does not
+//! use the wallet RPC API, so this crate can be used with wallet-disabled Bitcoin Core nodes.
+//!
+//! [`Emitter`] is the main structure which sources blockchain data from [`bitcoincore_rpc::Client`].
+//!
+//! To only get block updates (exclude mempool transactions), the caller can use
+//! [`Emitter::next_block`] or/and [`Emitter::next_header`] until it returns `Ok(None)` (which means
+//! the chain tip is reached). A separate method, [`Emitter::mempool`] can be used to emit the whole
+//! mempool.
+#![warn(missing_docs)]
+
+use bdk_chain::{local_chain::CheckPoint, BlockId};
+use bitcoin::{block::Header, Block, BlockHash, Transaction};
+pub use bitcoincore_rpc;
+use bitcoincore_rpc::bitcoincore_rpc_json;
+
+/// The [`Emitter`] is used to emit data sourced from [`bitcoincore_rpc::Client`].
+///
+/// Refer to [module-level documentation] for more.
+///
+/// [module-level documentation]: crate
+pub struct Emitter<'c, C> {
+    client: &'c C,
+    start_height: u32,
+
+    /// The checkpoint of the last-emitted block that is in the best chain. If it is later found
+    /// that the block is no longer in the best chain, it will be popped off from here.
+    last_cp: CheckPoint,
+
+    /// The block result returned from rpc of the last-emitted block. As this result contains the
+    /// next block's block hash (which we use to fetch the next block), we set this to `None`
+    /// whenever there are no more blocks, or the next block is no longer in the best chain. This
+    /// gives us an opportunity to re-fetch this result.
+    last_block: Option<bitcoincore_rpc_json::GetBlockResult>,
+
+    /// The latest first-seen epoch of emitted mempool transactions. This is used to determine
+    /// whether a mempool transaction is already emitted.
+    last_mempool_time: usize,
+
+    /// The last emitted block during our last mempool emission. This is used to determine whether
+    /// there has been a reorg since our last mempool emission.
+    last_mempool_tip: Option<u32>,
+}
+
+impl<'c, C: bitcoincore_rpc::RpcApi> Emitter<'c, C> {
+    /// Construct a new [`Emitter`].
+    ///
+    /// `last_cp` informs the emitter of the chain we are starting off with. This way, the emitter
+    /// can start emission from a block that connects to the original chain.
+    ///
+    /// `start_height` starts emission from a given height (if there are no conflicts with the
+    /// original chain).
+    pub fn new(client: &'c C, last_cp: CheckPoint, start_height: u32) -> Self {
+        Self {
+            client,
+            start_height,
+            last_cp,
+            last_block: None,
+            last_mempool_time: 0,
+            last_mempool_tip: None,
+        }
+    }
+
+    /// Emit mempool transactions, alongside their first-seen unix timestamps.
+    ///
+    /// This method emits each transaction only once, unless we cannot guarantee the transaction's
+    /// ancestors are already emitted.
+    ///
+    /// To understand why, consider a receiver which filters transactions based on whether it
+    /// alters the UTXO set of tracked script pubkeys. If an emitted mempool transaction spends a
+    /// tracked UTXO which is confirmed at height `h`, but the receiver has only seen up to block
+    /// of height `h-1`, we want to re-emit this transaction until the receiver has seen the block
+    /// at height `h`.
+    pub fn mempool(&mut self) -> Result<Vec<(Transaction, u64)>, bitcoincore_rpc::Error> {
+        let client = self.client;
+
+        // This is the emitted tip height during the last mempool emission.
+        let prev_mempool_tip = self
+            .last_mempool_tip
+            // We use `start_height - 1` as we cannot guarantee that the block at
+            // `start_height` has been emitted.
+            .unwrap_or(self.start_height.saturating_sub(1));
+
+        // Mempool txs come with a timestamp of when the tx is introduced to the mempool. We keep
+        // track of the latest mempool tx's timestamp to determine whether we have seen a tx
+        // before. `prev_mempool_time` is the previous timestamp and `last_time` records what will
+        // be the new latest timestamp.
+        let prev_mempool_time = self.last_mempool_time;
+        let mut latest_time = prev_mempool_time;
+
+        let txs_to_emit = client
+            .get_raw_mempool_verbose()?
+            .into_iter()
+            .filter_map({
+                let latest_time = &mut latest_time;
+                move |(txid, tx_entry)| -> Option<Result<_, bitcoincore_rpc::Error>> {
+                    let tx_time = tx_entry.time as usize;
+                    if tx_time > *latest_time {
+                        *latest_time = tx_time;
+                    }
+
+                    // Avoid emitting transactions that are already emitted if we can guarantee
+                    // blocks containing ancestors are already emitted. The bitcoind rpc interface
+                    // provides us with the block height that the tx is introduced to the mempool.
+                    // If we have already emitted the block of height, we can assume that all
+                    // ancestor txs have been processed by the receiver.
+                    let is_already_emitted = tx_time <= prev_mempool_time;
+                    let is_within_height = tx_entry.height <= prev_mempool_tip as _;
+                    if is_already_emitted && is_within_height {
+                        return None;
+                    }
+
+                    let tx = match client.get_raw_transaction(&txid, None) {
+                        Ok(tx) => tx,
+                        // the tx is confirmed or evicted since `get_raw_mempool_verbose`
+                        Err(err) if err.is_not_found_error() => return None,
+                        Err(err) => return Some(Err(err)),
+                    };
+
+                    Some(Ok((tx, tx_time as u64)))
+                }
+            })
+            .collect::<Result<Vec<_>, _>>()?;
+
+        self.last_mempool_time = latest_time;
+        self.last_mempool_tip = Some(self.last_cp.height());
+
+        Ok(txs_to_emit)
+    }
+
+    /// Emit the next block height and header (if any).
+    pub fn next_header(&mut self) -> Result<Option<BlockEvent<Header>>, bitcoincore_rpc::Error> {
+        Ok(poll(self, |hash| self.client.get_block_header(hash))?
+            .map(|(checkpoint, block)| BlockEvent { block, checkpoint }))
+    }
+
+    /// Emit the next block height and block (if any).
+    pub fn next_block(&mut self) -> Result<Option<BlockEvent<Block>>, bitcoincore_rpc::Error> {
+        Ok(poll(self, |hash| self.client.get_block(hash))?
+            .map(|(checkpoint, block)| BlockEvent { block, checkpoint }))
+    }
+}
+
+/// A newly emitted block from [`Emitter`].
+#[derive(Debug)]
+pub struct BlockEvent<B> {
+    /// Either a full [`Block`] or [`Header`] of the new block.
+    pub block: B,
+
+    /// The checkpoint of the new block.
+    ///
+    /// A [`CheckPoint`] is a node of a linked list of [`BlockId`]s. This checkpoint is linked to
+    /// all [`BlockId`]s originally passed in [`Emitter::new`] as well as emitted blocks since then.
+    /// These blocks are guaranteed to be of the same chain.
+    ///
+    /// This is important as BDK structures require block-to-apply to be connected with another
+    /// block in the original chain.
+    pub checkpoint: CheckPoint,
+}
+
+impl<B> BlockEvent<B> {
+    /// The block height of this new block.
+    pub fn block_height(&self) -> u32 {
+        self.checkpoint.height()
+    }
+
+    /// The block hash of this new block.
+    pub fn block_hash(&self) -> BlockHash {
+        self.checkpoint.hash()
+    }
+
+    /// The [`BlockId`] of a previous block that this block connects to.
+    ///
+    /// This either returns a [`BlockId`] of a previously emitted block or from the chain we started
+    /// with (passed in as `last_cp` in [`Emitter::new`]).
+    ///
+    /// This value is derived from [`BlockEvent::checkpoint`].
+    pub fn connected_to(&self) -> BlockId {
+        match self.checkpoint.prev() {
+            Some(prev_cp) => prev_cp.block_id(),
+            // there is no previous checkpoint, so just connect with itself
+            None => self.checkpoint.block_id(),
+        }
+    }
+}
+
+enum PollResponse {
+    Block(bitcoincore_rpc_json::GetBlockResult),
+    NoMoreBlocks,
+    /// Fetched block is not in the best chain.
+    BlockNotInBestChain,
+    AgreementFound(bitcoincore_rpc_json::GetBlockResult, CheckPoint),
+    /// Force the genesis checkpoint down the receiver's throat.
+    AgreementPointNotFound(BlockHash),
+}
+
+fn poll_once<C>(emitter: &Emitter<C>) -> Result<PollResponse, bitcoincore_rpc::Error>
+where
+    C: bitcoincore_rpc::RpcApi,
+{
+    let client = emitter.client;
+
+    if let Some(last_res) = &emitter.last_block {
+        let next_hash = if last_res.height < emitter.start_height as _ {
+            // enforce start height
+            let next_hash = client.get_block_hash(emitter.start_height as _)?;
+            // make sure last emission is still in best chain
+            if client.get_block_hash(last_res.height as _)? != last_res.hash {
+                return Ok(PollResponse::BlockNotInBestChain);
+            }
+            next_hash
+        } else {
+            match last_res.nextblockhash {
+                None => return Ok(PollResponse::NoMoreBlocks),
+                Some(next_hash) => next_hash,
+            }
+        };
+
+        let res = client.get_block_info(&next_hash)?;
+        if res.confirmations < 0 {
+            return Ok(PollResponse::BlockNotInBestChain);
+        }
+
+        return Ok(PollResponse::Block(res));
+    }
+
+    for cp in emitter.last_cp.iter() {
+        let res = match client.get_block_info(&cp.hash()) {
+            // block not in best chain
+            Ok(res) if res.confirmations < 0 => continue,
+            Ok(res) => res,
+            Err(e) if e.is_not_found_error() => {
+                if cp.height() > 0 {
+                    continue;
+                }
+                // if we can't find genesis block, we can't create an update that connects
+                break;
+            }
+            Err(e) => return Err(e),
+        };
+
+        // agreement point found
+        return Ok(PollResponse::AgreementFound(res, cp));
+    }
+
+    let genesis_hash = client.get_block_hash(0)?;
+    Ok(PollResponse::AgreementPointNotFound(genesis_hash))
+}
+
+fn poll<C, V, F>(
+    emitter: &mut Emitter<C>,
+    get_item: F,
+) -> Result<Option<(CheckPoint, V)>, bitcoincore_rpc::Error>
+where
+    C: bitcoincore_rpc::RpcApi,
+    F: Fn(&BlockHash) -> Result<V, bitcoincore_rpc::Error>,
+{
+    loop {
+        match poll_once(emitter)? {
+            PollResponse::Block(res) => {
+                let height = res.height as u32;
+                let hash = res.hash;
+                let item = get_item(&hash)?;
+
+                let new_cp = emitter
+                    .last_cp
+                    .clone()
+                    .push(BlockId { height, hash })
+                    .expect("must push");
+                emitter.last_cp = new_cp.clone();
+                emitter.last_block = Some(res);
+                return Ok(Some((new_cp, item)));
+            }
+            PollResponse::NoMoreBlocks => {
+                emitter.last_block = None;
+                return Ok(None);
+            }
+            PollResponse::BlockNotInBestChain => {
+                emitter.last_block = None;
+                continue;
+            }
+            PollResponse::AgreementFound(res, cp) => {
+                let agreement_h = res.height as u32;
+
+                // The tip during the last mempool emission needs to in the best chain, we reduce
+                // it if it is not.
+                if let Some(h) = emitter.last_mempool_tip.as_mut() {
+                    if *h > agreement_h {
+                        *h = agreement_h;
+                    }
+                }
+
+                // get rid of evicted blocks
+                emitter.last_cp = cp;
+                emitter.last_block = Some(res);
+                continue;
+            }
+            PollResponse::AgreementPointNotFound(genesis_hash) => {
+                emitter.last_cp = CheckPoint::new(BlockId {
+                    height: 0,
+                    hash: genesis_hash,
+                });
+                emitter.last_block = None;
+                continue;
+            }
+        }
+    }
+}
+
+/// Extends [`bitcoincore_rpc::Error`].
+pub trait BitcoindRpcErrorExt {
+    /// Returns whether the error is a "not found" error.
+    ///
+    /// This is useful since [`Emitter`] emits [`Result<_, bitcoincore_rpc::Error>`]s as
+    /// [`Iterator::Item`].
+    fn is_not_found_error(&self) -> bool;
+}
+
+impl BitcoindRpcErrorExt for bitcoincore_rpc::Error {
+    fn is_not_found_error(&self) -> bool {
+        if let bitcoincore_rpc::Error::JsonRpc(bitcoincore_rpc::jsonrpc::Error::Rpc(rpc_err)) = self
+        {
+            rpc_err.code == -5
+        } else {
+            false
+        }
+    }
+}

crates/bitcoind_rpc/tests/test_emitter.rs

@@ -0,0 +1,864 @@
+use std::collections::{BTreeMap, BTreeSet};
+
+use bdk_bitcoind_rpc::Emitter;
+use bdk_chain::{
+    bitcoin::{Address, Amount, BlockHash, Txid},
+    keychain::Balance,
+    local_chain::{self, CheckPoint, LocalChain},
+    Append, BlockId, IndexedTxGraph, SpkTxOutIndex,
+};
+use bitcoin::{
+    address::NetworkChecked, block::Header, hash_types::TxMerkleNode, hashes::Hash,
+    secp256k1::rand::random, Block, CompactTarget, OutPoint, ScriptBuf, ScriptHash, Transaction,
+    TxIn, TxOut, WScriptHash,
+};
+use bitcoincore_rpc::{
+    bitcoincore_rpc_json::{GetBlockTemplateModes, GetBlockTemplateRules},
+    RpcApi,
+};
+
+struct TestEnv {
+    #[allow(dead_code)]
+    daemon: bitcoind::BitcoinD,
+    client: bitcoincore_rpc::Client,
+}
+
+impl TestEnv {
+    fn new() -> anyhow::Result<Self> {
+        let daemon = match std::env::var_os("TEST_BITCOIND") {
+            Some(bitcoind_path) => bitcoind::BitcoinD::new(bitcoind_path),
+            None => bitcoind::BitcoinD::from_downloaded(),
+        }?;
+        let client = bitcoincore_rpc::Client::new(
+            &daemon.rpc_url(),
+            bitcoincore_rpc::Auth::CookieFile(daemon.params.cookie_file.clone()),
+        )?;
+        Ok(Self { daemon, client })
+    }
+
+    fn mine_blocks(
+        &self,
+        count: usize,
+        address: Option<Address>,
+    ) -> anyhow::Result<Vec<BlockHash>> {
+        let coinbase_address = match address {
+            Some(address) => address,
+            None => self.client.get_new_address(None, None)?.assume_checked(),
+        };
+        let block_hashes = self
+            .client
+            .generate_to_address(count as _, &coinbase_address)?;
+        Ok(block_hashes)
+    }
+
+    fn mine_empty_block(&self) -> anyhow::Result<(usize, BlockHash)> {
+        let bt = self.client.get_block_template(
+            GetBlockTemplateModes::Template,
+            &[GetBlockTemplateRules::SegWit],
+            &[],
+        )?;
+
+        let txdata = vec![Transaction {
+            version: 1,
+            lock_time: bitcoin::absolute::LockTime::from_height(0)?,
+            input: vec![TxIn {
+                previous_output: bitcoin::OutPoint::default(),
+                script_sig: ScriptBuf::builder()
+                    .push_int(bt.height as _)
+                    // randomn number so that re-mining creates unique block
+                    .push_int(random())
+                    .into_script(),
+                sequence: bitcoin::Sequence::default(),
+                witness: bitcoin::Witness::new(),
+            }],
+            output: vec![TxOut {
+                value: 0,
+                script_pubkey: ScriptBuf::new_p2sh(&ScriptHash::all_zeros()),
+            }],
+        }];
+
+        let bits: [u8; 4] = bt
+            .bits
+            .clone()
+            .try_into()
+            .expect("rpc provided us with invalid bits");
+
+        let mut block = Block {
+            header: Header {
+                version: bitcoin::block::Version::default(),
+                prev_blockhash: bt.previous_block_hash,
+                merkle_root: TxMerkleNode::all_zeros(),
+                time: Ord::max(bt.min_time, std::time::UNIX_EPOCH.elapsed()?.as_secs()) as u32,
+                bits: CompactTarget::from_consensus(u32::from_be_bytes(bits)),
+                nonce: 0,
+            },
+            txdata,
+        };
+
+        block.header.merkle_root = block.compute_merkle_root().expect("must compute");
+
+        for nonce in 0..=u32::MAX {
+            block.header.nonce = nonce;
+            if block.header.target().is_met_by(block.block_hash()) {
+                break;
+            }
+        }
+
+        self.client.submit_block(&block)?;
+        Ok((bt.height as usize, block.block_hash()))
+    }
+
+    fn invalidate_blocks(&self, count: usize) -> anyhow::Result<()> {
+        let mut hash = self.client.get_best_block_hash()?;
+        for _ in 0..count {
+            let prev_hash = self.client.get_block_info(&hash)?.previousblockhash;
+            self.client.invalidate_block(&hash)?;
+            match prev_hash {
+                Some(prev_hash) => hash = prev_hash,
+                None => break,
+            }
+        }
+        Ok(())
+    }
+
+    fn reorg(&self, count: usize) -> anyhow::Result<Vec<BlockHash>> {
+        let start_height = self.client.get_block_count()?;
+        self.invalidate_blocks(count)?;
+
+        let res = self.mine_blocks(count, None);
+        assert_eq!(
+            self.client.get_block_count()?,
+            start_height,
+            "reorg should not result in height change"
+        );
+        res
+    }
+
+    fn reorg_empty_blocks(&self, count: usize) -> anyhow::Result<Vec<(usize, BlockHash)>> {
+        let start_height = self.client.get_block_count()?;
+        self.invalidate_blocks(count)?;
+
+        let res = (0..count)
+            .map(|_| self.mine_empty_block())
+            .collect::<Result<Vec<_>, _>>()?;
+        assert_eq!(
+            self.client.get_block_count()?,
+            start_height,
+            "reorg should not result in height change"
+        );
+        Ok(res)
+    }
+
+    fn send(&self, address: &Address<NetworkChecked>, amount: Amount) -> anyhow::Result<Txid> {
+        let txid = self
+            .client
+            .send_to_address(address, amount, None, None, None, None, None, None)?;
+        Ok(txid)
+    }
+}
+
+/// Ensure that blocks are emitted in order even after reorg.
+///
+/// 1. Mine 101 blocks.
+/// 2. Emit blocks from [`Emitter`] and update the [`LocalChain`].
+/// 3. Reorg highest 6 blocks.
+/// 4. Emit blocks from [`Emitter`] and re-update the [`LocalChain`].
+#[test]
+pub fn test_sync_local_chain() -> anyhow::Result<()> {
+    let env = TestEnv::new()?;
+    let (mut local_chain, _) = LocalChain::from_genesis_hash(env.client.get_block_hash(0)?);
+    let mut emitter = Emitter::new(&env.client, local_chain.tip(), 0);
+
+    // mine some blocks and returned the actual block hashes
+    let exp_hashes = {
+        let mut hashes = vec![env.client.get_block_hash(0)?]; // include genesis block
+        hashes.extend(env.mine_blocks(101, None)?);
+        hashes
+    };
+
+    // see if the emitter outputs the right blocks
+    println!("first sync:");
+    while let Some(emission) = emitter.next_block()? {
+        let height = emission.block_height();
+        let hash = emission.block_hash();
+        assert_eq!(
+            emission.block_hash(),
+            exp_hashes[height as usize],
+            "emitted block hash is unexpected"
+        );
+
+        assert_eq!(
+            local_chain.apply_update(local_chain::Update {
+                tip: emission.checkpoint,
+                introduce_older_blocks: false,
+            })?,
+            BTreeMap::from([(height, Some(hash))]),
+            "chain update changeset is unexpected",
+        );
+    }
+
+    assert_eq!(
+        local_chain.blocks(),
+        &exp_hashes
+            .iter()
+            .enumerate()
+            .map(|(i, hash)| (i as u32, *hash))
+            .collect(),
+        "final local_chain state is unexpected",
+    );
+
+    // perform reorg
+    let reorged_blocks = env.reorg(6)?;
+    let exp_hashes = exp_hashes
+        .iter()
+        .take(exp_hashes.len() - reorged_blocks.len())
+        .chain(&reorged_blocks)
+        .cloned()
+        .collect::<Vec<_>>();
+
+    // see if the emitter outputs the right blocks
+    println!("after reorg:");
+    let mut exp_height = exp_hashes.len() - reorged_blocks.len();
+    while let Some(emission) = emitter.next_block()? {
+        let height = emission.block_height();
+        let hash = emission.block_hash();
+        assert_eq!(
+            height, exp_height as u32,
+            "emitted block has unexpected height"
+        );
+
+        assert_eq!(
+            hash, exp_hashes[height as usize],
+            "emitted block is unexpected"
+        );
+
+        assert_eq!(
+            local_chain.apply_update(local_chain::Update {
+                tip: emission.checkpoint,
+                introduce_older_blocks: false,
+            })?,
+            if exp_height == exp_hashes.len() - reorged_blocks.len() {
+                core::iter::once((height, Some(hash)))
+                    .chain((height + 1..exp_hashes.len() as u32).map(|h| (h, None)))
+                    .collect::<bdk_chain::local_chain::ChangeSet>()
+            } else {
+                BTreeMap::from([(height, Some(hash))])
+            },
+            "chain update changeset is unexpected",
+        );
+
+        exp_height += 1;
+    }
+
+    assert_eq!(
+        local_chain.blocks(),
+        &exp_hashes
+            .iter()
+            .enumerate()
+            .map(|(i, hash)| (i as u32, *hash))
+            .collect(),
+        "final local_chain state is unexpected after reorg",
+    );
+
+    Ok(())
+}
+
+/// Ensure that [`EmittedUpdate::into_tx_graph_update`] behaves appropriately for both mempool and
+/// block updates.
+///
+/// [`EmittedUpdate::into_tx_graph_update`]: bdk_bitcoind_rpc::EmittedUpdate::into_tx_graph_update
+#[test]
+fn test_into_tx_graph() -> anyhow::Result<()> {
+    let env = TestEnv::new()?;
+
+    println!("getting new addresses!");
+    let addr_0 = env.client.get_new_address(None, None)?.assume_checked();
+    let addr_1 = env.client.get_new_address(None, None)?.assume_checked();
+    let addr_2 = env.client.get_new_address(None, None)?.assume_checked();
+    println!("got new addresses!");
+
+    println!("mining block!");
+    env.mine_blocks(101, None)?;
+    println!("mined blocks!");
+
+    let (mut chain, _) = LocalChain::from_genesis_hash(env.client.get_block_hash(0)?);
+    let mut indexed_tx_graph = IndexedTxGraph::<BlockId, _>::new({
+        let mut index = SpkTxOutIndex::<usize>::default();
+        index.insert_spk(0, addr_0.script_pubkey());
+        index.insert_spk(1, addr_1.script_pubkey());
+        index.insert_spk(2, addr_2.script_pubkey());
+        index
+    });
+
+    let emitter = &mut Emitter::new(&env.client, chain.tip(), 0);
+
+    while let Some(emission) = emitter.next_block()? {
+        let height = emission.block_height();
+        let _ = chain.apply_update(local_chain::Update {
+            tip: emission.checkpoint,
+            introduce_older_blocks: false,
+        })?;
+        let indexed_additions = indexed_tx_graph.apply_block_relevant(&emission.block, height);
+        assert!(indexed_additions.is_empty());
+    }
+
+    // send 3 txs to a tracked address, these txs will be in the mempool
+    let exp_txids = {
+        let mut txids = BTreeSet::new();
+        for _ in 0..3 {
+            txids.insert(env.client.send_to_address(
+                &addr_0,
+                Amount::from_sat(10_000),
+                None,
+                None,
+                None,
+                None,
+                None,
+                None,
+            )?);
+        }
+        txids
+    };
+
+    // expect that the next block should be none and we should get 3 txs from mempool
+    {
+        // next block should be `None`
+        assert!(emitter.next_block()?.is_none());
+
+        let mempool_txs = emitter.mempool()?;
+        let indexed_additions = indexed_tx_graph.batch_insert_unconfirmed(mempool_txs);
+        assert_eq!(
+            indexed_additions
+                .graph
+                .txs
+                .iter()
+                .map(|tx| tx.txid())
+                .collect::<BTreeSet<Txid>>(),
+            exp_txids,
+            "changeset should have the 3 mempool transactions",
+        );
+        assert!(indexed_additions.graph.anchors.is_empty());
+    }
+
+    // mine a block that confirms the 3 txs
+    let exp_block_hash = env.mine_blocks(1, None)?[0];
+    let exp_block_height = env.client.get_block_info(&exp_block_hash)?.height as u32;
+    let exp_anchors = exp_txids
+        .iter()
+        .map({
+            let anchor = BlockId {
+                height: exp_block_height,
+                hash: exp_block_hash,
+            };
+            move |&txid| (anchor, txid)
+        })
+        .collect::<BTreeSet<_>>();
+
+    // must receive mined block which will confirm the transactions.
+    {
+        let emission = emitter.next_block()?.expect("must get mined block");
+        let height = emission.block_height();
+        let _ = chain.apply_update(local_chain::Update {
+            tip: emission.checkpoint,
+            introduce_older_blocks: false,
+        })?;
+        let indexed_additions = indexed_tx_graph.apply_block_relevant(&emission.block, height);
+        assert!(indexed_additions.graph.txs.is_empty());
+        assert!(indexed_additions.graph.txouts.is_empty());
+        assert_eq!(indexed_additions.graph.anchors, exp_anchors);
+    }
+
+    Ok(())
+}
+
+/// Ensure next block emitted after reorg is at reorg height.
+///
+/// After a reorg, if the last-emitted block height is equal or greater than the reorg height, and
+/// the fallback height is equal to or lower than the reorg height, the next block/header emission
+/// should be at the reorg height.
+///
+/// TODO: If the reorg height is lower than the fallback height, how do we find a block height to
+/// emit that can connect with our receiver chain?
+#[test]
+fn ensure_block_emitted_after_reorg_is_at_reorg_height() -> anyhow::Result<()> {
+    const EMITTER_START_HEIGHT: usize = 100;
+    const CHAIN_TIP_HEIGHT: usize = 110;
+
+    let env = TestEnv::new()?;
+    let mut emitter = Emitter::new(
+        &env.client,
+        CheckPoint::new(BlockId {
+            height: 0,
+            hash: env.client.get_block_hash(0)?,
+        }),
+        EMITTER_START_HEIGHT as _,
+    );
+
+    env.mine_blocks(CHAIN_TIP_HEIGHT, None)?;
+    while emitter.next_header()?.is_some() {}
+
+    for reorg_count in 1..=10 {
+        let replaced_blocks = env.reorg_empty_blocks(reorg_count)?;
+        let next_emission = emitter.next_header()?.expect("must emit block after reorg");
+        assert_eq!(
+            (
+                next_emission.block_height() as usize,
+                next_emission.block_hash()
+            ),
+            replaced_blocks[0],
+            "block emitted after reorg should be at the reorg height"
+        );
+        while emitter.next_header()?.is_some() {}
+    }
+
+    Ok(())
+}
+
+fn process_block(
+    recv_chain: &mut LocalChain,
+    recv_graph: &mut IndexedTxGraph<BlockId, SpkTxOutIndex<()>>,
+    block: Block,
+    block_height: u32,
+) -> anyhow::Result<()> {
+    recv_chain
+        .apply_update(CheckPoint::from_header(&block.header, block_height).into_update(false))?;
+    let _ = recv_graph.apply_block(block, block_height);
+    Ok(())
+}
+
+fn sync_from_emitter<C>(
+    recv_chain: &mut LocalChain,
+    recv_graph: &mut IndexedTxGraph<BlockId, SpkTxOutIndex<()>>,
+    emitter: &mut Emitter<C>,
+) -> anyhow::Result<()>
+where
+    C: bitcoincore_rpc::RpcApi,
+{
+    while let Some(emission) = emitter.next_block()? {
+        let height = emission.block_height();
+        process_block(recv_chain, recv_graph, emission.block, height)?;
+    }
+    Ok(())
+}
+
+fn get_balance(
+    recv_chain: &LocalChain,
+    recv_graph: &IndexedTxGraph<BlockId, SpkTxOutIndex<()>>,
+) -> anyhow::Result<Balance> {
+    let chain_tip = recv_chain.tip().block_id();
+    let outpoints = recv_graph.index.outpoints().clone();
+    let balance = recv_graph
+        .graph()
+        .balance(recv_chain, chain_tip, outpoints, |_, _| true);
+    Ok(balance)
+}
+
+/// If a block is reorged out, ensure that containing transactions that do not exist in the
+/// replacement block(s) become unconfirmed.
+#[test]
+fn tx_can_become_unconfirmed_after_reorg() -> anyhow::Result<()> {
+    const PREMINE_COUNT: usize = 101;
+    const ADDITIONAL_COUNT: usize = 11;
+    const SEND_AMOUNT: Amount = Amount::from_sat(10_000);
+
+    let env = TestEnv::new()?;
+    let mut emitter = Emitter::new(
+        &env.client,
+        CheckPoint::new(BlockId {
+            height: 0,
+            hash: env.client.get_block_hash(0)?,
+        }),
+        0,
+    );
+
+    // setup addresses
+    let addr_to_mine = env.client.get_new_address(None, None)?.assume_checked();
+    let spk_to_track = ScriptBuf::new_v0_p2wsh(&WScriptHash::all_zeros());
+    let addr_to_track = Address::from_script(&spk_to_track, bitcoin::Network::Regtest)?;
+
+    // setup receiver
+    let (mut recv_chain, _) = LocalChain::from_genesis_hash(env.client.get_block_hash(0)?);
+    let mut recv_graph = IndexedTxGraph::<BlockId, _>::new({
+        let mut recv_index = SpkTxOutIndex::default();
+        recv_index.insert_spk((), spk_to_track.clone());
+        recv_index
+    });
+
+    // mine and sync receiver up to tip
+    env.mine_blocks(PREMINE_COUNT, Some(addr_to_mine))?;
+
+    // create transactions that are tracked by our receiver
+    for _ in 0..ADDITIONAL_COUNT {
+        let txid = env.send(&addr_to_track, SEND_AMOUNT)?;
+
+        // lock outputs that send to `addr_to_track`
+        let outpoints_to_lock = env
+            .client
+            .get_transaction(&txid, None)?
+            .transaction()?
+            .output
+            .into_iter()
+            .enumerate()
+            .filter(|(_, txo)| txo.script_pubkey == spk_to_track)
+            .map(|(vout, _)| OutPoint::new(txid, vout as _))
+            .collect::<Vec<_>>();
+        env.client.lock_unspent(&outpoints_to_lock)?;
+
+        let _ = env.mine_blocks(1, None)?;
+    }
+
+    // get emitter up to tip
+    sync_from_emitter(&mut recv_chain, &mut recv_graph, &mut emitter)?;
+
+    assert_eq!(
+        get_balance(&recv_chain, &recv_graph)?,
+        Balance {
+            confirmed: SEND_AMOUNT.to_sat() * ADDITIONAL_COUNT as u64,
+            ..Balance::default()
+        },
+        "initial balance must be correct",
+    );
+
+    // perform reorgs with different depths
+    for reorg_count in 1..=ADDITIONAL_COUNT {
+        env.reorg_empty_blocks(reorg_count)?;
+        sync_from_emitter(&mut recv_chain, &mut recv_graph, &mut emitter)?;
+
+        assert_eq!(
+            get_balance(&recv_chain, &recv_graph)?,
+            Balance {
+                confirmed: SEND_AMOUNT.to_sat() * (ADDITIONAL_COUNT - reorg_count) as u64,
+                trusted_pending: SEND_AMOUNT.to_sat() * reorg_count as u64,
+                ..Balance::default()
+            },
+            "reorg_count: {}",
+            reorg_count,
+        );
+    }
+
+    Ok(())
+}
+
+/// Ensure avoid-re-emission-logic is sound when [`Emitter`] is synced to tip.
+///
+/// The receiver (bdk_chain structures) is synced to the chain tip, and there is txs in the mempool.
+/// When we call Emitter::mempool multiple times, mempool txs should not be re-emitted, even if the
+/// chain tip is extended.
+#[test]
+fn mempool_avoids_re_emission() -> anyhow::Result<()> {
+    const BLOCKS_TO_MINE: usize = 101;
+    const MEMPOOL_TX_COUNT: usize = 2;
+
+    let env = TestEnv::new()?;
+    let mut emitter = Emitter::new(
+        &env.client,
+        CheckPoint::new(BlockId {
+            height: 0,
+            hash: env.client.get_block_hash(0)?,
+        }),
+        0,
+    );
+
+    // mine blocks and sync up emitter
+    let addr = env.client.get_new_address(None, None)?.assume_checked();
+    env.mine_blocks(BLOCKS_TO_MINE, Some(addr.clone()))?;
+    while emitter.next_header()?.is_some() {}
+
+    // have some random txs in mempool
+    let exp_txids = (0..MEMPOOL_TX_COUNT)
+        .map(|_| env.send(&addr, Amount::from_sat(2100)))
+        .collect::<Result<BTreeSet<Txid>, _>>()?;
+
+    // the first emission should include all transactions
+    let emitted_txids = emitter
+        .mempool()?
+        .into_iter()
+        .map(|(tx, _)| tx.txid())
+        .collect::<BTreeSet<Txid>>();
+    assert_eq!(
+        emitted_txids, exp_txids,
+        "all mempool txs should be emitted"
+    );
+
+    // second emission should be empty
+    assert!(
+        emitter.mempool()?.is_empty(),
+        "second emission should be empty"
+    );
+
+    // mine empty blocks + sync up our emitter -> we should still not re-emit
+    for _ in 0..BLOCKS_TO_MINE {
+        env.mine_empty_block()?;
+    }
+    while emitter.next_header()?.is_some() {}
+    assert!(
+        emitter.mempool()?.is_empty(),
+        "third emission, after chain tip is extended, should also be empty"
+    );
+
+    Ok(())
+}
+
+/// Ensure mempool tx is still re-emitted if [`Emitter`] has not reached the tx's introduction
+/// height.
+///
+/// We introduce a mempool tx after each block, where blocks are empty (does not confirm previous
+/// mempool txs). Then we emit blocks from [`Emitter`] (intertwining `mempool` calls). We check
+/// that `mempool` should always re-emit txs that have introduced at a height greater than the last
+/// emitted block height.
+#[test]
+fn mempool_re_emits_if_tx_introduction_height_not_reached() -> anyhow::Result<()> {
+    const PREMINE_COUNT: usize = 101;
+    const MEMPOOL_TX_COUNT: usize = 21;
+
+    let env = TestEnv::new()?;
+    let mut emitter = Emitter::new(
+        &env.client,
+        CheckPoint::new(BlockId {
+            height: 0,
+            hash: env.client.get_block_hash(0)?,
+        }),
+        0,
+    );
+
+    // mine blocks to get initial balance, sync emitter up to tip
+    let addr = env.client.get_new_address(None, None)?.assume_checked();
+    env.mine_blocks(PREMINE_COUNT, Some(addr.clone()))?;
+    while emitter.next_header()?.is_some() {}
+
+    // mine blocks to introduce txs to mempool at different heights
+    let tx_introductions = (0..MEMPOOL_TX_COUNT)
+        .map(|_| -> anyhow::Result<_> {
+            let (height, _) = env.mine_empty_block()?;
+            let txid = env.send(&addr, Amount::from_sat(2100))?;
+            Ok((height, txid))
+        })
+        .collect::<anyhow::Result<BTreeSet<_>>>()?;
+
+    assert_eq!(
+        emitter
+            .mempool()?
+            .into_iter()
+            .map(|(tx, _)| tx.txid())
+            .collect::<BTreeSet<_>>(),
+        tx_introductions.iter().map(|&(_, txid)| txid).collect(),
+        "first mempool emission should include all txs",
+    );
+    assert_eq!(
+        emitter
+            .mempool()?
+            .into_iter()
+            .map(|(tx, _)| tx.txid())
+            .collect::<BTreeSet<_>>(),
+        tx_introductions.iter().map(|&(_, txid)| txid).collect(),
+        "second mempool emission should still include all txs",
+    );
+
+    // At this point, the emitter has seen all mempool transactions. It should only re-emit those
+    // that have introduction heights less than the emitter's last-emitted block tip.
+    while let Some(emission) = emitter.next_header()? {
+        let height = emission.block_height();
+        // We call `mempool()` twice.
+        // The second call (at height `h`) should skip the tx introduced at height `h`.
+        for try_index in 0..2 {
+            let exp_txids = tx_introductions
+                .range((height as usize + try_index, Txid::all_zeros())..)
+                .map(|&(_, txid)| txid)
+                .collect::<BTreeSet<_>>();
+            let emitted_txids = emitter
+                .mempool()?
+                .into_iter()
+                .map(|(tx, _)| tx.txid())
+                .collect::<BTreeSet<_>>();
+            assert_eq!(
+                emitted_txids, exp_txids,
+                "\n emission {} (try {}) must only contain txs introduced at that height or lower: \n\t missing: {:?} \n\t extra: {:?}",
+                height,
+                try_index,
+                exp_txids
+                    .difference(&emitted_txids)
+                    .map(|txid| (txid, tx_introductions.iter().find_map(|(h, id)| if id == txid { Some(h) } else { None }).unwrap()))
+                    .collect::<Vec<_>>(),
+                emitted_txids
+                    .difference(&exp_txids)
+                    .map(|txid| (txid, tx_introductions.iter().find_map(|(h, id)| if id == txid { Some(h) } else { None }).unwrap()))
+                    .collect::<Vec<_>>(),
+            );
+        }
+    }
+
+    Ok(())
+}
+
+/// Ensure we force re-emit all mempool txs after reorg.
+#[test]
+fn mempool_during_reorg() -> anyhow::Result<()> {
+    const TIP_DIFF: usize = 10;
+    const PREMINE_COUNT: usize = 101;
+
+    let env = TestEnv::new()?;
+    let mut emitter = Emitter::new(
+        &env.client,
+        CheckPoint::new(BlockId {
+            height: 0,
+            hash: env.client.get_block_hash(0)?,
+        }),
+        0,
+    );
+
+    // mine blocks to get initial balance
+    let addr = env.client.get_new_address(None, None)?.assume_checked();
+    env.mine_blocks(PREMINE_COUNT, Some(addr.clone()))?;
+
+    // introduce mempool tx at each block extension
+    for _ in 0..TIP_DIFF {
+        env.mine_empty_block()?;
+        env.send(&addr, Amount::from_sat(2100))?;
+    }
+
+    // sync emitter to tip, first mempool emission should include all txs (as we haven't emitted
+    // from the mempool yet)
+    while emitter.next_header()?.is_some() {}
+    assert_eq!(
+        emitter
+            .mempool()?
+            .into_iter()
+            .map(|(tx, _)| tx.txid())
+            .collect::<BTreeSet<_>>(),
+        env.client
+            .get_raw_mempool()?
+            .into_iter()
+            .collect::<BTreeSet<_>>(),
+        "first mempool emission should include all txs",
+    );
+
+    // perform reorgs at different heights, these reorgs will not confirm transactions in the
+    // mempool
+    for reorg_count in 1..TIP_DIFF {
+        println!("REORG COUNT: {}", reorg_count);
+        env.reorg_empty_blocks(reorg_count)?;
+
+        // This is a map of mempool txids to tip height where the tx was introduced to the mempool
+        // we recalculate this at every loop as reorgs may evict transactions from mempool. We use
+        // the introduction height to determine whether we expect a tx to appear in a mempool
+        // emission.
+        // TODO: How can have have reorg logic in `TestEnv` NOT blacklast old blocks first?
+        let tx_introductions = dbg!(env
+            .client
+            .get_raw_mempool_verbose()?
+            .into_iter()
+            .map(|(txid, entry)| (txid, entry.height as usize))
+            .collect::<BTreeMap<_, _>>());
+
+        // `next_header` emits the replacement block of the reorg
+        if let Some(emission) = emitter.next_header()? {
+            let height = emission.block_height();
+            println!("\t- replacement height: {}", height);
+
+            // the mempool emission (that follows the first block emission after reorg) should only
+            // include mempool txs introduced at reorg height or greater
+            let mempool = emitter
+                .mempool()?
+                .into_iter()
+                .map(|(tx, _)| tx.txid())
+                .collect::<BTreeSet<_>>();
+            let exp_mempool = tx_introductions
+                .iter()
+                .filter(|(_, &intro_h)| intro_h >= (height as usize))
+                .map(|(&txid, _)| txid)
+                .collect::<BTreeSet<_>>();
+            assert_eq!(
+                mempool, exp_mempool,
+                "the first mempool emission after reorg should only include mempool txs introduced at reorg height or greater"
+            );
+
+            let mempool = emitter
+                .mempool()?
+                .into_iter()
+                .map(|(tx, _)| tx.txid())
+                .collect::<BTreeSet<_>>();
+            let exp_mempool = tx_introductions
+                .iter()
+                .filter(|&(_, &intro_height)| intro_height > (height as usize))
+                .map(|(&txid, _)| txid)
+                .collect::<BTreeSet<_>>();
+            assert_eq!(
+                mempool, exp_mempool,
+                "following mempool emissions after reorg should exclude mempool introduction heights <= last emitted block height: \n\t missing: {:?} \n\t extra: {:?}",
+                exp_mempool
+                    .difference(&mempool)
+                    .map(|txid| (txid, tx_introductions.get(txid).unwrap()))
+                    .collect::<Vec<_>>(),
+                mempool
+                    .difference(&exp_mempool)
+                    .map(|txid| (txid, tx_introductions.get(txid).unwrap()))
+                    .collect::<Vec<_>>(),
+            );
+        }
+
+        // sync emitter to tip
+        while emitter.next_header()?.is_some() {}
+    }
+
+    Ok(())
+}
+
+/// If blockchain re-org includes the start height, emit new start height block
+///
+/// 1. mine 101 blocks
+/// 2. emit blocks 99a, 100a
+/// 3. invalidate blocks 99a, 100a, 101a
+/// 4. mine new blocks 99b, 100b, 101b
+/// 5. emit block 99b
+///
+/// The block hash of 99b should be different than 99a, but their previous block hashes should
+/// be the same.
+#[test]
+fn no_agreement_point() -> anyhow::Result<()> {
+    const PREMINE_COUNT: usize = 101;
+
+    let env = TestEnv::new()?;
+
+    // start height is 99
+    let mut emitter = Emitter::new(
+        &env.client,
+        CheckPoint::new(BlockId {
+            height: 0,
+            hash: env.client.get_block_hash(0)?,
+        }),
+        (PREMINE_COUNT - 2) as u32,
+    );
+
+    // mine 101 blocks
+    env.mine_blocks(PREMINE_COUNT, None)?;
+
+    // emit block 99a
+    let block_header_99a = emitter.next_header()?.expect("block 99a header").block;
+    let block_hash_99a = block_header_99a.block_hash();
+    let block_hash_98a = block_header_99a.prev_blockhash;
+
+    // emit block 100a
+    let block_header_100a = emitter.next_header()?.expect("block 100a header").block;
+    let block_hash_100a = block_header_100a.block_hash();
+
+    // get hash for block 101a
+    let block_hash_101a = env.client.get_block_hash(101)?;
+
+    // invalidate blocks 99a, 100a, 101a
+    env.client.invalidate_block(&block_hash_99a)?;
+    env.client.invalidate_block(&block_hash_100a)?;
+    env.client.invalidate_block(&block_hash_101a)?;
+
+    // mine new blocks 99b, 100b, 101b
+    env.mine_blocks(3, None)?;
+
+    // emit block header 99b
+    let block_header_99b = emitter.next_header()?.expect("block 99b header").block;
+    let block_hash_99b = block_header_99b.block_hash();
+    let block_hash_98b = block_header_99b.prev_blockhash;
+
+    assert_ne!(block_hash_99a, block_hash_99b);
+    assert_eq!(block_hash_98a, block_hash_98b);
+
+    Ok(())
+}

crates/chain/Cargo.toml

@@ -1,8 +1,8 @@
 [package]
 name = "bdk_chain"
-version = "0.5.0"
+version = "0.11.0"
 edition = "2021"
-rust-version = "1.57"
+rust-version = "1.63"
 homepage = "https://bitcoindevkit.org"
 repository = "https://github.com/bitcoindevkit/bdk"
 documentation = "https://docs.rs/bdk_chain"
@@ -14,13 +14,12 @@ readme = "README.md"
 
 [dependencies]
 # For no-std, remember to enable the bitcoin/no-std feature
-bitcoin = { version = "0.29", default-features = false }
+bitcoin = { version = "0.30.0", default-features = false }
 serde_crate = { package = "serde", version = "1", optional = true, features = ["derive"] }
 
 # Use hashbrown as a feature flag to have HashSet and HashMap from it.
-# note version 0.13 breaks outs MSRV.
-hashbrown = { version = "0.11", optional = true, features = ["serde"] }
-miniscript = { version = "9.0.0", optional = true, default-features = false }
+hashbrown = { version = "0.9.1", optional = true, features = ["serde"] }
+miniscript = { version = "10.0.0", optional = true, default-features = false }
 
 [dev-dependencies]
 rand = "0.8"

crates/chain/src/chain_data.rs

@@ -1,6 +1,6 @@
 use bitcoin::{hashes::Hash, BlockHash, OutPoint, TxOut, Txid};
 
-use crate::{Anchor, COINBASE_MATURITY};
+use crate::{Anchor, AnchorFromBlockPosition, COINBASE_MATURITY};
 
 /// Represents the observed position of some chain data.
 ///
@@ -9,7 +9,7 @@ use crate::{Anchor, COINBASE_MATURITY};
 pub enum ChainPosition<A> {
     /// The chain data is seen as confirmed, and in anchored by `A`.
     Confirmed(A),
-    /// The chain data is seen in mempool at this given timestamp.
+    /// The chain data is not confirmed and last seen in the mempool at this timestamp.
     Unconfirmed(u64),
 }
 
@@ -48,14 +48,14 @@ impl<A: Anchor> ChainPosition<A> {
     serde(crate = "serde_crate")
 )]
 pub enum ConfirmationTime {
-    /// The confirmed variant.
+    /// The transaction is confirmed
     Confirmed {
         /// Confirmation height.
         height: u32,
         /// Confirmation time in unix seconds.
         time: u64,
     },
-    /// The unconfirmed variant.
+    /// The transaction is unconfirmed
     Unconfirmed {
         /// The last-seen timestamp in unix seconds.
         last_seen: u64,
@@ -74,19 +74,22 @@ impl ConfirmationTime {
     }
 }
 
-impl From<ChainPosition<ConfirmationTimeAnchor>> for ConfirmationTime {
-    fn from(observed_as: ChainPosition<ConfirmationTimeAnchor>) -> Self {
+impl From<ChainPosition<ConfirmationTimeHeightAnchor>> for ConfirmationTime {
+    fn from(observed_as: ChainPosition<ConfirmationTimeHeightAnchor>) -> Self {
         match observed_as {
             ChainPosition::Confirmed(a) => Self::Confirmed {
                 height: a.confirmation_height,
                 time: a.confirmation_time,
             },
-            ChainPosition::Unconfirmed(_) => Self::Unconfirmed { last_seen: 0 },
+            ChainPosition::Unconfirmed(last_seen) => Self::Unconfirmed { last_seen },
         }
     }
 }
 
 /// A reference to a block in the canonical chain.
+///
+/// `BlockId` implements [`Anchor`]. When a transaction is anchored to `BlockId`, the confirmation
+/// block and anchor block are the same block.
 #[derive(Debug, Clone, PartialEq, Eq, Copy, PartialOrd, Ord, core::hash::Hash)]
 #[cfg_attr(
     feature = "serde",
@@ -100,11 +103,23 @@ pub struct BlockId {
     pub hash: BlockHash,
 }
 
+impl Anchor for BlockId {
+    fn anchor_block(&self) -> Self {
+        *self
+    }
+}
+
+impl AnchorFromBlockPosition for BlockId {
+    fn from_block_position(_block: &bitcoin::Block, block_id: BlockId, _tx_pos: usize) -> Self {
+        block_id
+    }
+}
+
 impl Default for BlockId {
     fn default() -> Self {
         Self {
             height: Default::default(),
-            hash: BlockHash::from_inner([0u8; 32]),
+            hash: BlockHash::all_zeros(),
         }
     }
 }
@@ -131,6 +146,10 @@ impl From<(&u32, &BlockHash)> for BlockId {
 }
 
 /// An [`Anchor`] implementation that also records the exact confirmation height of the transaction.
+///
+/// Note that the confirmation block and the anchor block can be different here.
+///
+/// Refer to [`Anchor`] for more details.
 #[derive(Debug, Default, Clone, PartialEq, Eq, Copy, PartialOrd, Ord, core::hash::Hash)]
 #[cfg_attr(
     feature = "serde",
@@ -138,13 +157,12 @@ impl From<(&u32, &BlockHash)> for BlockId {
     serde(crate = "serde_crate")
 )]
 pub struct ConfirmationHeightAnchor {
-    /// The anchor block.
-    pub anchor_block: BlockId,
-
     /// The exact confirmation height of the transaction.
     ///
     /// It is assumed that this value is never larger than the height of the anchor block.
     pub confirmation_height: u32,
+    /// The anchor block.
+    pub anchor_block: BlockId,
 }
 
 impl Anchor for ConfirmationHeightAnchor {
@@ -157,24 +175,37 @@ impl Anchor for ConfirmationHeightAnchor {
     }
 }
 
+impl AnchorFromBlockPosition for ConfirmationHeightAnchor {
+    fn from_block_position(_block: &bitcoin::Block, block_id: BlockId, _tx_pos: usize) -> Self {
+        Self {
+            anchor_block: block_id,
+            confirmation_height: block_id.height,
+        }
+    }
+}
+
 /// An [`Anchor`] implementation that also records the exact confirmation time and height of the
 /// transaction.
+///
+/// Note that the confirmation block and the anchor block can be different here.
+///
+/// Refer to [`Anchor`] for more details.
 #[derive(Debug, Default, Clone, PartialEq, Eq, Copy, PartialOrd, Ord, core::hash::Hash)]
 #[cfg_attr(
     feature = "serde",
     derive(serde::Deserialize, serde::Serialize),
     serde(crate = "serde_crate")
 )]
-pub struct ConfirmationTimeAnchor {
+pub struct ConfirmationTimeHeightAnchor {
+    /// The confirmation height of the transaction being anchored.
+    pub confirmation_height: u32,
+    /// The confirmation time of the transaction being anchored.
+    pub confirmation_time: u64,
     /// The anchor block.
     pub anchor_block: BlockId,
-    /// The confirmation height of the chain data being anchored.
-    pub confirmation_height: u32,
-    /// The confirmation time of the chain data being anchored.
-    pub confirmation_time: u64,
 }
 
-impl Anchor for ConfirmationTimeAnchor {
+impl Anchor for ConfirmationTimeHeightAnchor {
     fn anchor_block(&self) -> BlockId {
         self.anchor_block
     }
@@ -183,15 +214,26 @@ impl Anchor for ConfirmationTimeAnchor {
         self.confirmation_height
     }
 }
+
+impl AnchorFromBlockPosition for ConfirmationTimeHeightAnchor {
+    fn from_block_position(block: &bitcoin::Block, block_id: BlockId, _tx_pos: usize) -> Self {
+        Self {
+            anchor_block: block_id,
+            confirmation_height: block_id.height,
+            confirmation_time: block.header.time as _,
+        }
+    }
+}
+
 /// A `TxOut` with as much data as we can retrieve about it
 #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
 pub struct FullTxOut<A> {
+    /// The position of the transaction in `outpoint` in the overall chain.
+    pub chain_position: ChainPosition<A>,
     /// The location of the `TxOut`.
     pub outpoint: OutPoint,
     /// The `TxOut`.
     pub txout: TxOut,
-    /// The position of the transaction in `outpoint` in the overall chain.
-    pub chain_position: ChainPosition<A>,
     /// The txid and chain position of the transaction (if any) that has spent this output.
     pub spent_by: Option<(ChainPosition<A>, Txid)>,
     /// Whether this output is on a coinbase transaction.
@@ -202,7 +244,7 @@ impl<A: Anchor> FullTxOut<A> {
     /// Whether the `txout` is considered mature.
     ///
     /// Depending on the implementation of [`confirmation_height_upper_bound`] in [`Anchor`], this
-    /// method may return false-negatives. In other words, interpretted confirmation count may be
+    /// method may return false-negatives. In other words, interpreted confirmation count may be
     /// less than the actual value.
     ///
     /// [`confirmation_height_upper_bound`]: Anchor::confirmation_height_upper_bound
@@ -226,10 +268,10 @@ impl<A: Anchor> FullTxOut<A> {
 
     /// Whether the utxo is/was/will be spendable with chain `tip`.
     ///
-    /// This method does not take into account the locktime.
+    /// This method does not take into account the lock time.
     ///
     /// Depending on the implementation of [`confirmation_height_upper_bound`] in [`Anchor`], this
-    /// method may return false-negatives. In other words, interpretted confirmation count may be
+    /// method may return false-negatives. In other words, interpreted confirmation count may be
     /// less than the actual value.
     ///
     /// [`confirmation_height_upper_bound`]: Anchor::confirmation_height_upper_bound
@@ -256,3 +298,35 @@ impl<A: Anchor> FullTxOut<A> {
         true
     }
 }
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn chain_position_ord() {
+        let unconf1 = ChainPosition::<ConfirmationHeightAnchor>::Unconfirmed(10);
+        let unconf2 = ChainPosition::<ConfirmationHeightAnchor>::Unconfirmed(20);
+        let conf1 = ChainPosition::Confirmed(ConfirmationHeightAnchor {
+            confirmation_height: 9,
+            anchor_block: BlockId {
+                height: 20,
+                ..Default::default()
+            },
+        });
+        let conf2 = ChainPosition::Confirmed(ConfirmationHeightAnchor {
+            confirmation_height: 12,
+            anchor_block: BlockId {
+                height: 15,
+                ..Default::default()
+            },
+        });
+
+        assert!(unconf2 > unconf1, "higher last_seen means higher ord");
+        assert!(unconf1 > conf1, "unconfirmed is higher ord than confirmed");
+        assert!(
+            conf2 > conf1,
+            "confirmation_height is higher then it should be higher ord"
+        );
+    }
+}

crates/chain/src/chain_oracle.rs

@@ -3,7 +3,7 @@ use crate::BlockId;
 /// Represents a service that tracks the blockchain.
 ///
 /// The main method is [`is_block_in_chain`] which determines whether a given block of [`BlockId`]
-/// is an ancestor of another "static block".
+/// is an ancestor of the `chain_tip`.
 ///
 /// [`is_block_in_chain`]: Self::is_block_in_chain
 pub trait ChainOracle {
@@ -21,5 +21,5 @@ pub trait ChainOracle {
     ) -> Result<Option<bool>, Self::Error>;
 
     /// Get the best chain's chain tip.
-    fn get_chain_tip(&self) -> Result<Option<BlockId>, Self::Error>;
+    fn get_chain_tip(&self) -> Result<BlockId, Self::Error>;
 }

crates/chain/src/descriptor_ext.rs

@@ -3,12 +3,14 @@ use crate::miniscript::{Descriptor, DescriptorPublicKey};
 /// A trait to extend the functionality of a miniscript descriptor.
 pub trait DescriptorExt {
     /// Returns the minimum value (in satoshis) at which an output is broadcastable.
+    /// Panics if the descriptor wildcard is hardened.
     fn dust_value(&self) -> u64;
 }
 
 impl DescriptorExt for Descriptor<DescriptorPublicKey> {
     fn dust_value(&self) -> u64 {
         self.at_derivation_index(0)
+            .expect("descriptor can't have hardened derivation")
             .script_pubkey()
             .dust_value()
             .to_sat()

crates/chain/src/indexed_tx_graph.rs

@@ -1,19 +1,17 @@
-//! Contains the [`IndexedTxGraph`] structure and associated types.
-//!
-//! This is essentially a [`TxGraph`] combined with an indexer.
-
+//! Contains the [`IndexedTxGraph`] and associated types. Refer to the
+//! [`IndexedTxGraph`] documentation for more.
 use alloc::vec::Vec;
-use bitcoin::{OutPoint, Transaction, TxOut};
+use bitcoin::{Block, OutPoint, Transaction, TxOut, Txid};
 
 use crate::{
-    keychain::DerivationAdditions,
-    tx_graph::{Additions, TxGraph},
-    Anchor, Append,
+    keychain,
+    tx_graph::{self, TxGraph},
+    Anchor, AnchorFromBlockPosition, Append, BlockId,
 };
 
-/// A struct that combines [`TxGraph`] and an [`Indexer`] implementation.
+/// The [`IndexedTxGraph`] combines a [`TxGraph`] and an [`Indexer`] implementation.
 ///
-/// This structure ensures that [`TxGraph`] and [`Indexer`] are updated atomically.
+/// It ensures that [`TxGraph`] and [`Indexer`] are updated atomically.
 #[derive(Debug)]
 pub struct IndexedTxGraph<A, I> {
     /// Transaction index.
@@ -46,128 +44,233 @@ impl<A, I> IndexedTxGraph<A, I> {
 }
 
 impl<A: Anchor, I: Indexer> IndexedTxGraph<A, I> {
-    /// Applies the [`IndexedAdditions`] to the [`IndexedTxGraph`].
-    pub fn apply_additions(&mut self, additions: IndexedAdditions<A, I::Additions>) {
-        let IndexedAdditions {
-            graph_additions,
-            index_additions,
-        } = additions;
+    /// Applies the [`ChangeSet`] to the [`IndexedTxGraph`].
+    pub fn apply_changeset(&mut self, changeset: ChangeSet<A, I::ChangeSet>) {
+        self.index.apply_changeset(changeset.indexer);
 
-        self.index.apply_additions(index_additions);
-
-        for tx in &graph_additions.txs {
+        for tx in &changeset.graph.txs {
             self.index.index_tx(tx);
         }
-        for (&outpoint, txout) in &graph_additions.txouts {
+        for (&outpoint, txout) in &changeset.graph.txouts {
             self.index.index_txout(outpoint, txout);
         }
 
-        self.graph.apply_additions(graph_additions);
+        self.graph.apply_changeset(changeset.graph);
+    }
+
+    /// Determines the [`ChangeSet`] between `self` and an empty [`IndexedTxGraph`].
+    pub fn initial_changeset(&self) -> ChangeSet<A, I::ChangeSet> {
+        let graph = self.graph.initial_changeset();
+        let indexer = self.index.initial_changeset();
+        ChangeSet { graph, indexer }
     }
 }
 
 impl<A: Anchor, I: Indexer> IndexedTxGraph<A, I>
 where
-    I::Additions: Default + Append,
+    I::ChangeSet: Default + Append,
 {
+    fn index_tx_graph_changeset(
+        &mut self,
+        tx_graph_changeset: &tx_graph::ChangeSet<A>,
+    ) -> I::ChangeSet {
+        let mut changeset = I::ChangeSet::default();
+        for added_tx in &tx_graph_changeset.txs {
+            changeset.append(self.index.index_tx(added_tx));
+        }
+        for (&added_outpoint, added_txout) in &tx_graph_changeset.txouts {
+            changeset.append(self.index.index_txout(added_outpoint, added_txout));
+        }
+        changeset
+    }
+
     /// Apply an `update` directly.
     ///
-    /// `update` is a [`TxGraph<A>`] and the resultant changes is returned as [`IndexedAdditions`].
-    pub fn apply_update(&mut self, update: TxGraph<A>) -> IndexedAdditions<A, I::Additions> {
-        let graph_additions = self.graph.apply_update(update);
-
-        let mut index_additions = I::Additions::default();
-        for added_tx in &graph_additions.txs {
-            index_additions.append(self.index.index_tx(added_tx));
-        }
-        for (&added_outpoint, added_txout) in &graph_additions.txouts {
-            index_additions.append(self.index.index_txout(added_outpoint, added_txout));
-        }
-
-        IndexedAdditions {
-            graph_additions,
-            index_additions,
-        }
+    /// `update` is a [`TxGraph<A>`] and the resultant changes is returned as [`ChangeSet`].
+    pub fn apply_update(&mut self, update: TxGraph<A>) -> ChangeSet<A, I::ChangeSet> {
+        let graph = self.graph.apply_update(update);
+        let indexer = self.index_tx_graph_changeset(&graph);
+        ChangeSet { graph, indexer }
     }
 
     /// Insert a floating `txout` of given `outpoint`.
-    pub fn insert_txout(
-        &mut self,
-        outpoint: OutPoint,
-        txout: &TxOut,
-    ) -> IndexedAdditions<A, I::Additions> {
-        let mut update = TxGraph::<A>::default();
-        let _ = update.insert_txout(outpoint, txout.clone());
-        self.apply_update(update)
+    pub fn insert_txout(&mut self, outpoint: OutPoint, txout: TxOut) -> ChangeSet<A, I::ChangeSet> {
+        let graph = self.graph.insert_txout(outpoint, txout);
+        let indexer = self.index_tx_graph_changeset(&graph);
+        ChangeSet { graph, indexer }
     }
 
     /// Insert and index a transaction into the graph.
-    ///
-    /// `anchors` can be provided to anchor the transaction to various blocks. `seen_at` is a
-    /// unix timestamp of when the transaction is last seen.
-    pub fn insert_tx(
-        &mut self,
-        tx: &Transaction,
-        anchors: impl IntoIterator<Item = A>,
-        seen_at: Option<u64>,
-    ) -> IndexedAdditions<A, I::Additions> {
-        let txid = tx.txid();
-
-        let mut update = TxGraph::<A>::default();
-        if self.graph.get_tx(txid).is_none() {
-            let _ = update.insert_tx(tx.clone());
-        }
-        for anchor in anchors.into_iter() {
-            let _ = update.insert_anchor(txid, anchor);
-        }
-        if let Some(seen_at) = seen_at {
-            let _ = update.insert_seen_at(txid, seen_at);
-        }
-
-        self.apply_update(update)
+    pub fn insert_tx(&mut self, tx: Transaction) -> ChangeSet<A, I::ChangeSet> {
+        let graph = self.graph.insert_tx(tx);
+        let indexer = self.index_tx_graph_changeset(&graph);
+        ChangeSet { graph, indexer }
     }
 
-    /// Insert relevant transactions from the given `txs` iterator.
+    /// Insert an `anchor` for a given transaction.
+    pub fn insert_anchor(&mut self, txid: Txid, anchor: A) -> ChangeSet<A, I::ChangeSet> {
+        self.graph.insert_anchor(txid, anchor).into()
+    }
+
+    /// Insert a unix timestamp of when a transaction is seen in the mempool.
+    ///
+    /// This is used for transaction conflict resolution in [`TxGraph`] where the transaction with
+    /// the later last-seen is prioritized.
+    pub fn insert_seen_at(&mut self, txid: Txid, seen_at: u64) -> ChangeSet<A, I::ChangeSet> {
+        self.graph.insert_seen_at(txid, seen_at).into()
+    }
+
+    /// Batch insert transactions, filtering out those that are irrelevant.
     ///
     /// Relevancy is determined by the [`Indexer::is_tx_relevant`] implementation of `I`. Irrelevant
     /// transactions in `txs` will be ignored. `txs` do not need to be in topological order.
-    ///
-    /// `anchors` can be provided to anchor the transactions to blocks. `seen_at` is a unix
-    /// timestamp of when the transactions are last seen.
-    pub fn insert_relevant_txs<'t>(
+    pub fn batch_insert_relevant<'t>(
         &mut self,
         txs: impl IntoIterator<Item = (&'t Transaction, impl IntoIterator<Item = A>)>,
-        seen_at: Option<u64>,
-    ) -> IndexedAdditions<A, I::Additions> {
+    ) -> ChangeSet<A, I::ChangeSet> {
         // The algorithm below allows for non-topologically ordered transactions by using two loops.
         // This is achieved by:
         // 1. insert all txs into the index. If they are irrelevant then that's fine it will just
         //    not store anything about them.
         // 2. decide whether to insert them into the graph depending on whether `is_tx_relevant`
         //    returns true or not. (in a second loop).
-        let mut additions = IndexedAdditions::<A, I::Additions>::default();
-        let mut transactions = Vec::new();
-        for (tx, anchors) in txs.into_iter() {
-            additions.index_additions.append(self.index.index_tx(tx));
-            transactions.push((tx, anchors));
+        let txs = txs.into_iter().collect::<Vec<_>>();
+
+        let mut indexer = I::ChangeSet::default();
+        for (tx, _) in &txs {
+            indexer.append(self.index.index_tx(tx));
         }
-        additions.append(
-            transactions
-                .into_iter()
-                .filter_map(|(tx, anchors)| match self.index.is_tx_relevant(tx) {
-                    true => Some(self.insert_tx(tx, anchors, seen_at)),
-                    false => None,
-                })
-                .fold(Default::default(), |mut acc, other| {
-                    acc.append(other);
-                    acc
-                }),
+
+        let mut graph = tx_graph::ChangeSet::default();
+        for (tx, anchors) in txs {
+            if self.index.is_tx_relevant(tx) {
+                let txid = tx.txid();
+                graph.append(self.graph.insert_tx(tx.clone()));
+                for anchor in anchors {
+                    graph.append(self.graph.insert_anchor(txid, anchor));
+                }
+            }
+        }
+
+        ChangeSet { graph, indexer }
+    }
+
+    /// Batch insert unconfirmed transactions, filtering out those that are irrelevant.
+    ///
+    /// Relevancy is determined by the internal [`Indexer::is_tx_relevant`] implementation of `I`.
+    /// Irrelevant transactions in `txs` will be ignored.
+    ///
+    /// Items of `txs` are tuples containing the transaction and a *last seen* timestamp. The
+    /// *last seen* communicates when the transaction is last seen in the mempool which is used for
+    /// conflict-resolution in [`TxGraph`] (refer to [`TxGraph::insert_seen_at`] for details).
+    pub fn batch_insert_relevant_unconfirmed<'t>(
+        &mut self,
+        unconfirmed_txs: impl IntoIterator<Item = (&'t Transaction, u64)>,
+    ) -> ChangeSet<A, I::ChangeSet> {
+        // The algorithm below allows for non-topologically ordered transactions by using two loops.
+        // This is achieved by:
+        // 1. insert all txs into the index. If they are irrelevant then that's fine it will just
+        //    not store anything about them.
+        // 2. decide whether to insert them into the graph depending on whether `is_tx_relevant`
+        //    returns true or not. (in a second loop).
+        let txs = unconfirmed_txs.into_iter().collect::<Vec<_>>();
+
+        let mut indexer = I::ChangeSet::default();
+        for (tx, _) in &txs {
+            indexer.append(self.index.index_tx(tx));
+        }
+
+        let graph = self.graph.batch_insert_unconfirmed(
+            txs.into_iter()
+                .filter(|(tx, _)| self.index.is_tx_relevant(tx))
+                .map(|(tx, seen_at)| (tx.clone(), seen_at)),
         );
-        additions
+
+        ChangeSet { graph, indexer }
+    }
+
+    /// Batch insert unconfirmed transactions.
+    ///
+    /// Items of `txs` are tuples containing the transaction and a *last seen* timestamp. The
+    /// *last seen* communicates when the transaction is last seen in the mempool which is used for
+    /// conflict-resolution in [`TxGraph`] (refer to [`TxGraph::insert_seen_at`] for details).
+    ///
+    /// To filter out irrelevant transactions, use [`batch_insert_relevant_unconfirmed`] instead.
+    ///
+    /// [`batch_insert_relevant_unconfirmed`]: IndexedTxGraph::batch_insert_relevant_unconfirmed
+    pub fn batch_insert_unconfirmed(
+        &mut self,
+        txs: impl IntoIterator<Item = (Transaction, u64)>,
+    ) -> ChangeSet<A, I::ChangeSet> {
+        let graph = self.graph.batch_insert_unconfirmed(txs);
+        let indexer = self.index_tx_graph_changeset(&graph);
+        ChangeSet { graph, indexer }
     }
 }
 
-/// A structure that represents changes to an [`IndexedTxGraph`].
+/// Methods are available if the anchor (`A`) implements [`AnchorFromBlockPosition`].
+impl<A: Anchor, I: Indexer> IndexedTxGraph<A, I>
+where
+    I::ChangeSet: Default + Append,
+    A: AnchorFromBlockPosition,
+{
+    /// Batch insert all transactions of the given `block` of `height`, filtering out those that are
+    /// irrelevant.
+    ///
+    /// Each inserted transaction's anchor will be constructed from
+    /// [`AnchorFromBlockPosition::from_block_position`].
+    ///
+    /// Relevancy is determined by the internal [`Indexer::is_tx_relevant`] implementation of `I`.
+    /// Irrelevant transactions in `txs` will be ignored.
+    pub fn apply_block_relevant(
+        &mut self,
+        block: &Block,
+        height: u32,
+    ) -> ChangeSet<A, I::ChangeSet> {
+        let block_id = BlockId {
+            hash: block.block_hash(),
+            height,
+        };
+        let mut changeset = ChangeSet::<A, I::ChangeSet>::default();
+        for (tx_pos, tx) in block.txdata.iter().enumerate() {
+            changeset.indexer.append(self.index.index_tx(tx));
+            if self.index.is_tx_relevant(tx) {
+                let txid = tx.txid();
+                let anchor = A::from_block_position(block, block_id, tx_pos);
+                changeset.graph.append(self.graph.insert_tx(tx.clone()));
+                changeset
+                    .graph
+                    .append(self.graph.insert_anchor(txid, anchor));
+            }
+        }
+        changeset
+    }
+
+    /// Batch insert all transactions of the given `block` of `height`.
+    ///
+    /// Each inserted transaction's anchor will be constructed from
+    /// [`AnchorFromBlockPosition::from_block_position`].
+    ///
+    /// To only insert relevant transactions, use [`apply_block_relevant`] instead.
+    ///
+    /// [`apply_block_relevant`]: IndexedTxGraph::apply_block_relevant
+    pub fn apply_block(&mut self, block: Block, height: u32) -> ChangeSet<A, I::ChangeSet> {
+        let block_id = BlockId {
+            hash: block.block_hash(),
+            height,
+        };
+        let mut graph = tx_graph::ChangeSet::default();
+        for (tx_pos, tx) in block.txdata.iter().enumerate() {
+            let anchor = A::from_block_position(&block, block_id, tx_pos);
+            graph.append(self.graph.insert_anchor(tx.txid(), anchor));
+            graph.append(self.graph.insert_tx(tx.clone()));
+        }
+        let indexer = self.index_tx_graph_changeset(&graph);
+        ChangeSet { graph, indexer }
+    }
+}
+
+/// Represents changes to an [`IndexedTxGraph`].
 #[derive(Clone, Debug, PartialEq)]
 #[cfg_attr(
     feature = "serde",
@@ -181,64 +284,70 @@ where
     )
 )]
 #[must_use]
-pub struct IndexedAdditions<A, IA> {
-    /// [`TxGraph`] additions.
-    pub graph_additions: Additions<A>,
-    /// [`Indexer`] additions.
-    pub index_additions: IA,
+pub struct ChangeSet<A, IA> {
+    /// [`TxGraph`] changeset.
+    pub graph: tx_graph::ChangeSet<A>,
+    /// [`Indexer`] changeset.
+    pub indexer: IA,
 }
 
-impl<A, IA: Default> Default for IndexedAdditions<A, IA> {
+impl<A, IA: Default> Default for ChangeSet<A, IA> {
     fn default() -> Self {
         Self {
-            graph_additions: Default::default(),
-            index_additions: Default::default(),
+            graph: Default::default(),
+            indexer: Default::default(),
         }
     }
 }
 
-impl<A: Anchor, IA: Append> Append for IndexedAdditions<A, IA> {
+impl<A: Anchor, IA: Append> Append for ChangeSet<A, IA> {
     fn append(&mut self, other: Self) {
-        self.graph_additions.append(other.graph_additions);
-        self.index_additions.append(other.index_additions);
+        self.graph.append(other.graph);
+        self.indexer.append(other.indexer);
     }
 
     fn is_empty(&self) -> bool {
-        self.graph_additions.is_empty() && self.index_additions.is_empty()
+        self.graph.is_empty() && self.indexer.is_empty()
     }
 }
 
-impl<A, IA: Default> From<Additions<A>> for IndexedAdditions<A, IA> {
-    fn from(graph_additions: Additions<A>) -> Self {
+impl<A, IA: Default> From<tx_graph::ChangeSet<A>> for ChangeSet<A, IA> {
+    fn from(graph: tx_graph::ChangeSet<A>) -> Self {
         Self {
-            graph_additions,
+            graph,
             ..Default::default()
         }
     }
 }
 
-impl<A, K> From<DerivationAdditions<K>> for IndexedAdditions<A, DerivationAdditions<K>> {
-    fn from(index_additions: DerivationAdditions<K>) -> Self {
+impl<A, K> From<keychain::ChangeSet<K>> for ChangeSet<A, keychain::ChangeSet<K>> {
+    fn from(indexer: keychain::ChangeSet<K>) -> Self {
         Self {
-            graph_additions: Default::default(),
-            index_additions,
+            graph: Default::default(),
+            indexer,
         }
     }
 }
 
-/// Represents a structure that can index transaction data.
+/// Utilities for indexing transaction data.
+///
+/// Types which implement this trait can be used to construct an [`IndexedTxGraph`].
+/// This trait's methods should rarely be called directly.
 pub trait Indexer {
-    /// The resultant "additions" when new transaction data is indexed.
-    type Additions;
+    /// The resultant "changeset" when new transaction data is indexed.
+    type ChangeSet;
 
     /// Scan and index the given `outpoint` and `txout`.
-    fn index_txout(&mut self, outpoint: OutPoint, txout: &TxOut) -> Self::Additions;
+    fn index_txout(&mut self, outpoint: OutPoint, txout: &TxOut) -> Self::ChangeSet;
 
-    /// Scan and index the given transaction.
-    fn index_tx(&mut self, tx: &Transaction) -> Self::Additions;
+    /// Scans a transaction for relevant outpoints, which are stored and indexed internally.
+    fn index_tx(&mut self, tx: &Transaction) -> Self::ChangeSet;
 
-    /// Apply additions to itself.
-    fn apply_additions(&mut self, additions: Self::Additions);
+    /// Apply changeset to itself.
+    fn apply_changeset(&mut self, changeset: Self::ChangeSet);
+
+    /// Determines the [`ChangeSet`] between `self` and an empty [`Indexer`].
+    fn initial_changeset(&self) -> Self::ChangeSet;
 
     /// Determines whether the transaction should be included in the index.
     fn is_tx_relevant(&self, tx: &Transaction) -> bool;

crates/chain/src/keychain.rs

@@ -10,13 +10,7 @@
 //!
 //! [`SpkTxOutIndex`]: crate::SpkTxOutIndex
 
-use crate::{
-    collections::BTreeMap,
-    indexed_tx_graph::IndexedAdditions,
-    local_chain::{self, LocalChain},
-    tx_graph::TxGraph,
-    Anchor, Append,
-};
+use crate::{collections::BTreeMap, Append};
 
 #[cfg(feature = "miniscript")]
 mod txout_index;
@@ -24,12 +18,13 @@ mod txout_index;
 pub use txout_index::*;
 
 /// Represents updates to the derivation index of a [`KeychainTxOutIndex`].
+/// It maps each keychain `K` to its last revealed index.
 ///
-/// It can be applied to [`KeychainTxOutIndex`] with [`apply_additions`]. [`DerivationAdditions] are
+/// It can be applied to [`KeychainTxOutIndex`] with [`apply_changeset`]. [`ChangeSet`]s are
 /// monotone in that they will never decrease the revealed derivation index.
 ///
 /// [`KeychainTxOutIndex`]: crate::keychain::KeychainTxOutIndex
-/// [`apply_additions`]: crate::keychain::KeychainTxOutIndex::apply_additions
+/// [`apply_changeset`]: crate::keychain::KeychainTxOutIndex::apply_changeset
 #[derive(Clone, Debug, PartialEq)]
 #[cfg_attr(
     feature = "serde",
@@ -43,22 +38,17 @@ pub use txout_index::*;
     )
 )]
 #[must_use]
-pub struct DerivationAdditions<K>(pub BTreeMap<K, u32>);
-
-impl<K> DerivationAdditions<K> {
-    /// Returns whether the additions are empty.
-    pub fn is_empty(&self) -> bool {
-        self.0.is_empty()
-    }
+pub struct ChangeSet<K>(pub BTreeMap<K, u32>);
 
+impl<K> ChangeSet<K> {
     /// Get the inner map of the keychain to its new derivation index.
     pub fn as_inner(&self) -> &BTreeMap<K, u32> {
         &self.0
     }
 }
 
-impl<K: Ord> Append for DerivationAdditions<K> {
-    /// Append another [`DerivationAdditions`] into self.
+impl<K: Ord> Append for ChangeSet<K> {
+    /// Append another [`ChangeSet`] into self.
     ///
     /// If the keychain already exists, increase the index when the other's index > self's index.
     /// If the keychain did not exist, append the new keychain.
@@ -68,110 +58,29 @@ impl<K: Ord> Append for DerivationAdditions<K> {
                 *index = other_index.max(*index);
             }
         });
-
-        self.0.append(&mut other.0);
+        // We use `extend` instead of `BTreeMap::append` due to performance issues with `append`.
+        // Refer to https://github.com/rust-lang/rust/issues/34666#issuecomment-675658420
+        self.0.extend(other.0);
     }
 
+    /// Returns whether the changeset are empty.
     fn is_empty(&self) -> bool {
         self.0.is_empty()
     }
 }
 
-impl<K> Default for DerivationAdditions<K> {
+impl<K> Default for ChangeSet<K> {
     fn default() -> Self {
         Self(Default::default())
     }
 }
 
-impl<K> AsRef<BTreeMap<K, u32>> for DerivationAdditions<K> {
+impl<K> AsRef<BTreeMap<K, u32>> for ChangeSet<K> {
     fn as_ref(&self) -> &BTreeMap<K, u32> {
         &self.0
     }
 }
 
-/// A structure to update [`KeychainTxOutIndex`], [`TxGraph`] and [`LocalChain`]
-/// atomically.
-#[derive(Debug, Clone, PartialEq)]
-pub struct LocalUpdate<K, A> {
-    /// Last active derivation index per keychain (`K`).
-    pub keychain: BTreeMap<K, u32>,
-    /// Update for the [`TxGraph`].
-    pub graph: TxGraph<A>,
-    /// Update for the [`LocalChain`].
-    pub chain: LocalChain,
-}
-
-impl<K, A> Default for LocalUpdate<K, A> {
-    fn default() -> Self {
-        Self {
-            keychain: Default::default(),
-            graph: Default::default(),
-            chain: Default::default(),
-        }
-    }
-}
-
-/// A structure that records the corresponding changes as result of applying an [`LocalUpdate`].
-#[derive(Debug, Clone, PartialEq)]
-#[cfg_attr(
-    feature = "serde",
-    derive(serde::Deserialize, serde::Serialize),
-    serde(
-        crate = "serde_crate",
-        bound(
-            deserialize = "K: Ord + serde::Deserialize<'de>, A: Ord + serde::Deserialize<'de>",
-            serialize = "K: Ord + serde::Serialize, A: Ord + serde::Serialize",
-        )
-    )
-)]
-pub struct LocalChangeSet<K, A> {
-    /// Changes to the [`LocalChain`].
-    pub chain_changeset: local_chain::ChangeSet,
-
-    /// Additions to [`IndexedTxGraph`].
-    ///
-    /// [`IndexedTxGraph`]: crate::indexed_tx_graph::IndexedTxGraph
-    pub indexed_additions: IndexedAdditions<A, DerivationAdditions<K>>,
-}
-
-impl<K, A> Default for LocalChangeSet<K, A> {
-    fn default() -> Self {
-        Self {
-            chain_changeset: Default::default(),
-            indexed_additions: Default::default(),
-        }
-    }
-}
-
-impl<K: Ord, A: Anchor> Append for LocalChangeSet<K, A> {
-    fn append(&mut self, other: Self) {
-        Append::append(&mut self.chain_changeset, other.chain_changeset);
-        Append::append(&mut self.indexed_additions, other.indexed_additions);
-    }
-
-    fn is_empty(&self) -> bool {
-        self.chain_changeset.is_empty() && self.indexed_additions.is_empty()
-    }
-}
-
-impl<K, A> From<local_chain::ChangeSet> for LocalChangeSet<K, A> {
-    fn from(chain_changeset: local_chain::ChangeSet) -> Self {
-        Self {
-            chain_changeset,
-            ..Default::default()
-        }
-    }
-}
-
-impl<K, A> From<IndexedAdditions<A, DerivationAdditions<K>>> for LocalChangeSet<K, A> {
-    fn from(indexed_additions: IndexedAdditions<A, DerivationAdditions<K>>) -> Self {
-        Self {
-            indexed_additions,
-            ..Default::default()
-        }
-    }
-}
-
 /// Balance, differentiated into various categories.
 #[derive(Debug, PartialEq, Eq, Clone, Default)]
 #[cfg_attr(
@@ -250,8 +159,8 @@ mod test {
         lhs_di.insert(Keychain::Three, 3);
         rhs_di.insert(Keychain::Four, 4);
 
-        let mut lhs = DerivationAdditions(lhs_di);
-        let rhs = DerivationAdditions(rhs_di);
+        let mut lhs = ChangeSet(lhs_di);
+        let rhs = ChangeSet(rhs_di);
         lhs.append(rhs);
 
         // Exiting index doesn't update if the new index in `other` is lower than `self`.

crates/chain/src/keychain/txout_index.rs

@@ -3,27 +3,58 @@ use crate::{
     indexed_tx_graph::Indexer,
     miniscript::{Descriptor, DescriptorPublicKey},
     spk_iter::BIP32_MAX_INDEX,
-    ForEachTxOut, SpkIterator, SpkTxOutIndex,
+    SpkIterator, SpkTxOutIndex,
+};
+use bitcoin::{OutPoint, Script, Transaction, TxOut, Txid};
+use core::{
+    fmt::Debug,
+    ops::{Bound, RangeBounds},
 };
-use alloc::vec::Vec;
-use bitcoin::{OutPoint, Script, TxOut};
-use core::{fmt::Debug, ops::Deref};
 
 use crate::Append;
 
-use super::DerivationAdditions;
+const DEFAULT_LOOKAHEAD: u32 = 25;
 
-/// A convenient wrapper around [`SpkTxOutIndex`] that relates script pubkeys to miniscript public
-/// [`Descriptor`]s.
+/// [`KeychainTxOutIndex`] controls how script pubkeys are revealed for multiple keychains, and
+/// indexes [`TxOut`]s with them.
 ///
-/// Descriptors are referenced by the provided keychain generic (`K`).
+/// A single keychain is a chain of script pubkeys derived from a single [`Descriptor`]. Keychains
+/// are identified using the `K` generic. Script pubkeys are identified by the keychain that they
+/// are derived from `K`, as well as the derivation index `u32`.
 ///
-/// Script pubkeys for a descriptor are revealed chronologically from index 0. I.e., If the last
-/// revealed index of a descriptor is 5; scripts of indices 0 to 4 are guaranteed to be already
-/// revealed. In addition to revealed scripts, we have a `lookahead` parameter for each keychain,
-/// which defines the number of script pubkeys to store ahead of the last revealed index.
+/// # Revealed script pubkeys
 ///
-/// Methods that could update the last revealed index will return [`DerivationAdditions`] to report
+/// Tracking how script pubkeys are revealed is useful for collecting chain data. For example, if
+/// the user has requested 5 script pubkeys (to receive money with), we only need to use those
+/// script pubkeys to scan for chain data.
+///
+/// Call [`reveal_to_target`] or [`reveal_next_spk`] to reveal more script pubkeys.
+/// Call [`revealed_keychain_spks`] or [`revealed_spks`] to iterate through revealed script pubkeys.
+///
+/// # Lookahead script pubkeys
+///
+/// When an user first recovers a wallet (i.e. from a recovery phrase and/or descriptor), we will
+/// NOT have knowledge of which script pubkeys are revealed. So when we index a transaction or
+/// txout (using [`index_tx`]/[`index_txout`]) we scan the txouts against script pubkeys derived
+/// above the last revealed index. These additionally-derived script pubkeys are called the
+/// lookahead.
+///
+/// The [`KeychainTxOutIndex`] is constructed with the `lookahead` and cannot be altered. The
+/// default `lookahead` count is 1000. Use [`new`] to set a custom `lookahead`.
+///
+/// # Unbounded script pubkey iterator
+///
+/// For script-pubkey-based chain sources (such as Electrum/Esplora), an initial scan is best done
+/// by iterating though derived script pubkeys one by one and requesting transaction histories for
+/// each script pubkey. We will stop after x-number of script pubkeys have empty histories. An
+/// unbounded script pubkey iterator is useful to pass to such a chain source.
+///
+/// Call [`unbounded_spk_iter`] to get an unbounded script pubkey iterator for a given keychain.
+/// Call [`all_unbounded_spk_iters`] to get unbounded script pubkey iterators for all keychains.
+///
+/// # Change sets
+///
+/// Methods that can update the last revealed index will return [`super::ChangeSet`] to report
 /// these changes. This can be persisted for future recovery.
 ///
 /// ## Synopsis
@@ -48,7 +79,7 @@ use super::DerivationAdditions;
 /// # let secp = bdk_chain::bitcoin::secp256k1::Secp256k1::signing_only();
 /// # let (external_descriptor,_) = Descriptor::<DescriptorPublicKey>::parse_descriptor(&secp, "tr([73c5da0a/86'/0'/0']xprv9xgqHN7yz9MwCkxsBPN5qetuNdQSUttZNKw1dcYTV4mkaAFiBVGQziHs3NRSWMkCzvgjEe3n9xV8oYywvM8at9yRqyaZVz6TYYhX98VjsUk/0/*)").unwrap();
 /// # let (internal_descriptor,_) = Descriptor::<DescriptorPublicKey>::parse_descriptor(&secp, "tr([73c5da0a/86'/0'/0']xprv9xgqHN7yz9MwCkxsBPN5qetuNdQSUttZNKw1dcYTV4mkaAFiBVGQziHs3NRSWMkCzvgjEe3n9xV8oYywvM8at9yRqyaZVz6TYYhX98VjsUk/1/*)").unwrap();
-/// # let descriptor_for_user_42 = external_descriptor.clone();
+/// # let (descriptor_for_user_42, _) = Descriptor::<DescriptorPublicKey>::parse_descriptor(&secp, "tr([73c5da0a/86'/0'/0']xprv9xgqHN7yz9MwCkxsBPN5qetuNdQSUttZNKw1dcYTV4mkaAFiBVGQziHs3NRSWMkCzvgjEe3n9xV8oYywvM8at9yRqyaZVz6TYYhX98VjsUk/2/*)").unwrap();
 /// txout_index.add_keychain(MyKeychain::External, external_descriptor);
 /// txout_index.add_keychain(MyKeychain::Internal, internal_descriptor);
 /// txout_index.add_keychain(MyKeychain::MyAppUser { user_id: 42 }, descriptor_for_user_42);
@@ -59,6 +90,15 @@ use super::DerivationAdditions;
 /// [`Ord`]: core::cmp::Ord
 /// [`SpkTxOutIndex`]: crate::spk_txout_index::SpkTxOutIndex
 /// [`Descriptor`]: crate::miniscript::Descriptor
+/// [`reveal_to_target`]: KeychainTxOutIndex::reveal_to_target
+/// [`reveal_next_spk`]: KeychainTxOutIndex::reveal_next_spk
+/// [`revealed_keychain_spks`]: KeychainTxOutIndex::revealed_keychain_spks
+/// [`revealed_spks`]: KeychainTxOutIndex::revealed_spks
+/// [`index_tx`]: KeychainTxOutIndex::index_tx
+/// [`index_txout`]: KeychainTxOutIndex::index_txout
+/// [`new`]: KeychainTxOutIndex::new
+/// [`unbounded_spk_iter`]: KeychainTxOutIndex::unbounded_spk_iter
+/// [`all_unbounded_spk_iters`]: KeychainTxOutIndex::all_unbounded_spk_iters
 #[derive(Clone, Debug)]
 pub struct KeychainTxOutIndex<K> {
     inner: SpkTxOutIndex<(K, u32)>,
@@ -67,82 +107,73 @@ pub struct KeychainTxOutIndex<K> {
     // last revealed indexes
     last_revealed: BTreeMap<K, u32>,
     // lookahead settings for each keychain
-    lookahead: BTreeMap<K, u32>,
+    lookahead: u32,
 }
 
 impl<K> Default for KeychainTxOutIndex<K> {
     fn default() -> Self {
-        Self {
-            inner: SpkTxOutIndex::default(),
-            keychains: BTreeMap::default(),
-            last_revealed: BTreeMap::default(),
-            lookahead: BTreeMap::default(),
-        }
-    }
-}
-
-impl<K> Deref for KeychainTxOutIndex<K> {
-    type Target = SpkTxOutIndex<(K, u32)>;
-
-    fn deref(&self) -> &Self::Target {
-        &self.inner
+        Self::new(DEFAULT_LOOKAHEAD)
     }
 }
 
 impl<K: Clone + Ord + Debug> Indexer for KeychainTxOutIndex<K> {
-    type Additions = DerivationAdditions<K>;
+    type ChangeSet = super::ChangeSet<K>;
 
-    fn index_txout(&mut self, outpoint: OutPoint, txout: &TxOut) -> Self::Additions {
-        self.scan_txout(outpoint, txout)
-    }
-
-    fn index_tx(&mut self, tx: &bitcoin::Transaction) -> Self::Additions {
-        self.scan(tx)
-    }
-
-    fn apply_additions(&mut self, additions: Self::Additions) {
-        self.apply_additions(additions)
-    }
-
-    fn is_tx_relevant(&self, tx: &bitcoin::Transaction) -> bool {
-        self.is_relevant(tx)
-    }
-}
-
-impl<K: Clone + Ord + Debug> KeychainTxOutIndex<K> {
-    /// Scans an object for relevant outpoints, which are stored and indexed internally.
-    ///
-    /// If the matched script pubkey is part of the lookahead, the last stored index is updated for
-    /// the script pubkey's keychain and the [`DerivationAdditions`] returned will reflect the
-    /// change.
-    ///
-    /// Typically, this method is used in two situations:
-    ///
-    /// 1. After loading transaction data from the disk, you may scan over all the txouts to restore all
-    /// your txouts.
-    /// 2. When getting new data from the chain, you usually scan it before incorporating it into
-    /// your chain state (i.e., `SparseChain`, `ChainGraph`).
-    ///
-    /// See [`ForEachTxout`] for the types that support this.
-    ///
-    /// [`ForEachTxout`]: crate::ForEachTxOut
-    pub fn scan(&mut self, txouts: &impl ForEachTxOut) -> DerivationAdditions<K> {
-        let mut additions = DerivationAdditions::<K>::default();
-        txouts.for_each_txout(|(op, txout)| additions.append(self.scan_txout(op, txout)));
-        additions
-    }
-
-    /// Scan a single outpoint for a matching script pubkey.
-    ///
-    /// If it matches, this will store and index it.
-    pub fn scan_txout(&mut self, op: OutPoint, txout: &TxOut) -> DerivationAdditions<K> {
-        match self.inner.scan_txout(op, txout).cloned() {
+    fn index_txout(&mut self, outpoint: OutPoint, txout: &TxOut) -> Self::ChangeSet {
+        match self.inner.scan_txout(outpoint, txout).cloned() {
             Some((keychain, index)) => self.reveal_to_target(&keychain, index).1,
-            None => DerivationAdditions::default(),
+            None => super::ChangeSet::default(),
         }
     }
 
+    fn index_tx(&mut self, tx: &bitcoin::Transaction) -> Self::ChangeSet {
+        let mut changeset = super::ChangeSet::<K>::default();
+        for (op, txout) in tx.output.iter().enumerate() {
+            changeset.append(self.index_txout(OutPoint::new(tx.txid(), op as u32), txout));
+        }
+        changeset
+    }
+
+    fn initial_changeset(&self) -> Self::ChangeSet {
+        super::ChangeSet(self.last_revealed.clone())
+    }
+
+    fn apply_changeset(&mut self, changeset: Self::ChangeSet) {
+        self.apply_changeset(changeset)
+    }
+
+    fn is_tx_relevant(&self, tx: &bitcoin::Transaction) -> bool {
+        self.inner.is_relevant(tx)
+    }
+}
+
+impl<K> KeychainTxOutIndex<K> {
+    /// Construct a [`KeychainTxOutIndex`] with the given `lookahead`.
+    ///
+    /// The `lookahead` is the number of script pubkeys to derive and cache from the internal
+    /// descriptors over and above the last revealed script index. Without a lookahead the index
+    /// will miss outputs you own when processing transactions whose output script pubkeys lie
+    /// beyond the last revealed index. In certain situations, such as when performing an initial
+    /// scan of the blockchain during wallet import, it may be uncertain or unknown what the index
+    /// of the last revealed script pubkey actually is.
+    ///
+    /// Refer to [struct-level docs](KeychainTxOutIndex) for more about `lookahead`.
+    pub fn new(lookahead: u32) -> Self {
+        Self {
+            inner: SpkTxOutIndex::default(),
+            keychains: BTreeMap::new(),
+            last_revealed: BTreeMap::new(),
+            lookahead,
+        }
+    }
+}
+
+/// Methods that are *re-exposed* from the internal [`SpkTxOutIndex`].
+impl<K: Clone + Ord + Debug> KeychainTxOutIndex<K> {
     /// Return a reference to the internal [`SpkTxOutIndex`].
+    ///
+    /// **WARNING:** The internal index will contain lookahead spks. Refer to
+    /// [struct-level docs](KeychainTxOutIndex) for more about `lookahead`.
     pub fn inner(&self) -> &SpkTxOutIndex<(K, u32)> {
         &self.inner
     }
@@ -152,7 +183,116 @@ impl<K: Clone + Ord + Debug> KeychainTxOutIndex<K> {
         self.inner.outpoints()
     }
 
-    /// Return a reference to the internal map of the keychain to descriptors.
+    /// Iterate over known txouts that spend to tracked script pubkeys.
+    pub fn txouts(
+        &self,
+    ) -> impl DoubleEndedIterator<Item = (K, u32, OutPoint, &TxOut)> + ExactSizeIterator {
+        self.inner
+            .txouts()
+            .map(|((k, i), op, txo)| (k.clone(), *i, op, txo))
+    }
+
+    /// Finds all txouts on a transaction that has previously been scanned and indexed.
+    pub fn txouts_in_tx(
+        &self,
+        txid: Txid,
+    ) -> impl DoubleEndedIterator<Item = (K, u32, OutPoint, &TxOut)> {
+        self.inner
+            .txouts_in_tx(txid)
+            .map(|((k, i), op, txo)| (k.clone(), *i, op, txo))
+    }
+
+    /// Return the [`TxOut`] of `outpoint` if it has been indexed.
+    ///
+    /// The associated keychain and keychain index of the txout's spk is also returned.
+    ///
+    /// This calls [`SpkTxOutIndex::txout`] internally.
+    pub fn txout(&self, outpoint: OutPoint) -> Option<(K, u32, &TxOut)> {
+        self.inner
+            .txout(outpoint)
+            .map(|((k, i), txo)| (k.clone(), *i, txo))
+    }
+
+    /// Return the script that exists under the given `keychain`'s `index`.
+    ///
+    /// This calls [`SpkTxOutIndex::spk_at_index`] internally.
+    pub fn spk_at_index(&self, keychain: K, index: u32) -> Option<&Script> {
+        self.inner.spk_at_index(&(keychain, index))
+    }
+
+    /// Returns the keychain and keychain index associated with the spk.
+    ///
+    /// This calls [`SpkTxOutIndex::index_of_spk`] internally.
+    pub fn index_of_spk(&self, script: &Script) -> Option<(K, u32)> {
+        self.inner.index_of_spk(script).cloned()
+    }
+
+    /// Returns whether the spk under the `keychain`'s `index` has been used.
+    ///
+    /// Here, "unused" means that after the script pubkey was stored in the index, the index has
+    /// never scanned a transaction output with it.
+    ///
+    /// This calls [`SpkTxOutIndex::is_used`] internally.
+    pub fn is_used(&self, keychain: K, index: u32) -> bool {
+        self.inner.is_used(&(keychain, index))
+    }
+
+    /// Marks the script pubkey at `index` as used even though the tracker hasn't seen an output
+    /// with it.
+    ///
+    /// This only has an effect when the `index` had been added to `self` already and was unused.
+    ///
+    /// Returns whether the `index` was initially present as `unused`.
+    ///
+    /// This is useful when you want to reserve a script pubkey for something but don't want to add
+    /// the transaction output using it to the index yet. Other callers will consider `index` on
+    /// `keychain` used until you call [`unmark_used`].
+    ///
+    /// This calls [`SpkTxOutIndex::mark_used`] internally.
+    ///
+    /// [`unmark_used`]: Self::unmark_used
+    pub fn mark_used(&mut self, keychain: K, index: u32) -> bool {
+        self.inner.mark_used(&(keychain, index))
+    }
+
+    /// Undoes the effect of [`mark_used`]. Returns whether the `index` is inserted back into
+    /// `unused`.
+    ///
+    /// Note that if `self` has scanned an output with this script pubkey, then this will have no
+    /// effect.
+    ///
+    /// This calls [`SpkTxOutIndex::unmark_used`] internally.
+    ///
+    /// [`mark_used`]: Self::mark_used
+    pub fn unmark_used(&mut self, keychain: K, index: u32) -> bool {
+        self.inner.unmark_used(&(keychain, index))
+    }
+
+    /// Computes total input value going from script pubkeys in the index (sent) and the total output
+    /// value going to script pubkeys in the index (received) in `tx`. For the `sent` to be computed
+    /// correctly, the output being spent must have already been scanned by the index. Calculating
+    /// received just uses the [`Transaction`] outputs directly, so it will be correct even if it has
+    /// not been scanned.
+    ///
+    /// This calls [`SpkTxOutIndex::sent_and_received`] internally.
+    pub fn sent_and_received(&self, tx: &Transaction) -> (u64, u64) {
+        self.inner.sent_and_received(tx)
+    }
+
+    /// Computes the net value that this transaction gives to the script pubkeys in the index and
+    /// *takes* from the transaction outputs in the index. Shorthand for calling
+    /// [`sent_and_received`] and subtracting sent from received.
+    ///
+    /// This calls [`SpkTxOutIndex::net_value`] internally.
+    ///
+    /// [`sent_and_received`]: Self::sent_and_received
+    pub fn net_value(&self, tx: &Transaction) -> i64 {
+        self.inner.net_value(tx)
+    }
+}
+
+impl<K: Clone + Ord + Debug> KeychainTxOutIndex<K> {
+    /// Return a reference to the internal map of keychain to descriptors.
     pub fn keychains(&self) -> &BTreeMap<K, Descriptor<DescriptorPublicKey>> {
         &self.keychains
     }
@@ -168,80 +308,43 @@ impl<K: Clone + Ord + Debug> KeychainTxOutIndex<K> {
     pub fn add_keychain(&mut self, keychain: K, descriptor: Descriptor<DescriptorPublicKey>) {
         let old_descriptor = &*self
             .keychains
-            .entry(keychain)
+            .entry(keychain.clone())
             .or_insert_with(|| descriptor.clone());
         assert_eq!(
             &descriptor, old_descriptor,
             "keychain already contains a different descriptor"
         );
+        self.replenish_lookahead(&keychain, self.lookahead);
     }
 
-    /// Return the lookahead setting for each keychain.
+    /// Get the lookahead setting.
     ///
-    /// Refer to [`set_lookahead`] for a deeper explanation of the `lookahead`.
+    /// Refer to [`new`] for more information on the `lookahead`.
     ///
-    /// [`set_lookahead`]: Self::set_lookahead
-    pub fn lookaheads(&self) -> &BTreeMap<K, u32> {
-        &self.lookahead
+    /// [`new`]: Self::new
+    pub fn lookahead(&self) -> u32 {
+        self.lookahead
     }
 
-    /// Convenience method to call [`set_lookahead`] for all keychains.
+    /// Store lookahead scripts until `target_index` (inclusive).
     ///
-    /// [`set_lookahead`]: Self::set_lookahead
-    pub fn set_lookahead_for_all(&mut self, lookahead: u32) {
-        for keychain in &self.keychains.keys().cloned().collect::<Vec<_>>() {
-            self.lookahead.insert(keychain.clone(), lookahead);
-            self.replenish_lookahead(keychain);
-        }
-    }
-
-    /// Set the lookahead count for `keychain`.
-    ///
-    /// The lookahead is the number of scripts to cache ahead of the last stored script index. This
-    /// is useful during a scan via [`scan`] or [`scan_txout`].
-    ///
-    /// # Panics
-    ///
-    /// This will panic if the `keychain` does not exist.
-    ///
-    /// [`scan`]: Self::scan
-    /// [`scan_txout`]: Self::scan_txout
-    pub fn set_lookahead(&mut self, keychain: &K, lookahead: u32) {
-        self.lookahead.insert(keychain.clone(), lookahead);
-        self.replenish_lookahead(keychain);
-    }
-
-    /// Convenience method to call [`lookahead_to_target`] for multiple keychains.
-    ///
-    /// [`lookahead_to_target`]: Self::lookahead_to_target
-    pub fn lookahead_to_target_multi(&mut self, target_indexes: BTreeMap<K, u32>) {
-        for (keychain, target_index) in target_indexes {
-            self.lookahead_to_target(&keychain, target_index)
-        }
-    }
-
-    /// Store lookahead scripts until `target_index`.
-    ///
-    /// This does not change the `lookahead` setting.
+    /// This does not change the global `lookahead` setting.
     pub fn lookahead_to_target(&mut self, keychain: &K, target_index: u32) {
-        let next_index = self.next_store_index(keychain);
-        if let Some(temp_lookahead) = target_index.checked_sub(next_index).filter(|&v| v > 0) {
-            let old_lookahead = self.lookahead.insert(keychain.clone(), temp_lookahead);
-            self.replenish_lookahead(keychain);
+        let (next_index, _) = self.next_index(keychain);
 
-            // revert
-            match old_lookahead {
-                Some(lookahead) => self.lookahead.insert(keychain.clone(), lookahead),
-                None => self.lookahead.remove(keychain),
-            };
+        let temp_lookahead = (target_index + 1)
+            .checked_sub(next_index)
+            .filter(|&index| index > 0);
+
+        if let Some(temp_lookahead) = temp_lookahead {
+            self.replenish_lookahead(keychain, temp_lookahead);
         }
     }
 
-    fn replenish_lookahead(&mut self, keychain: &K) {
+    fn replenish_lookahead(&mut self, keychain: &K, lookahead: u32) {
         let descriptor = self.keychains.get(keychain).expect("keychain must exist");
         let next_store_index = self.next_store_index(keychain);
         let next_reveal_index = self.last_revealed.get(keychain).map_or(0, |v| *v + 1);
-        let lookahead = self.lookahead.get(keychain).map_or(0, |v| *v);
 
         for (new_index, new_spk) in
             SpkIterator::new_with_range(descriptor, next_store_index..next_reveal_index + lookahead)
@@ -256,64 +359,74 @@ impl<K: Clone + Ord + Debug> KeychainTxOutIndex<K> {
     fn next_store_index(&self, keychain: &K) -> u32 {
         self.inner()
             .all_spks()
+            // This range is filtering out the spks with a keychain different than
+            // `keychain`. We don't use filter here as range is more optimized.
             .range((keychain.clone(), u32::MIN)..(keychain.clone(), u32::MAX))
             .last()
-            .map_or(0, |((_, v), _)| *v + 1)
+            .map_or(0, |((_, index), _)| *index + 1)
     }
 
-    /// Generates script pubkey iterators for every `keychain`. The iterators iterate over all
-    /// derivable script pubkeys.
-    pub fn spks_of_all_keychains(
+    /// Get an unbounded spk iterator over a given `keychain`.
+    ///
+    /// # Panics
+    ///
+    /// This will panic if the given `keychain`'s descriptor does not exist.
+    pub fn unbounded_spk_iter(&self, keychain: &K) -> SpkIterator<Descriptor<DescriptorPublicKey>> {
+        SpkIterator::new(
+            self.keychains
+                .get(keychain)
+                .expect("keychain does not exist")
+                .clone(),
+        )
+    }
+
+    /// Get unbounded spk iterators for all keychains.
+    pub fn all_unbounded_spk_iters(
         &self,
     ) -> BTreeMap<K, SpkIterator<Descriptor<DescriptorPublicKey>>> {
         self.keychains
             .iter()
-            .map(|(keychain, descriptor)| {
-                (
-                    keychain.clone(),
-                    SpkIterator::new_with_range(descriptor.clone(), 0..),
-                )
-            })
+            .map(|(k, descriptor)| (k.clone(), SpkIterator::new(descriptor.clone())))
             .collect()
     }
 
-    /// Generates a script pubkey iterator for the given `keychain`'s descriptor (if it exists). The
-    /// iterator iterates over all derivable scripts of the keychain's descriptor.
-    ///
-    /// # Panics
-    ///
-    /// This will panic if the `keychain` does not exist.
-    pub fn spks_of_keychain(&self, keychain: &K) -> SpkIterator<Descriptor<DescriptorPublicKey>> {
-        let descriptor = self
-            .keychains
-            .get(keychain)
-            .expect("keychain must exist")
-            .clone();
-        SpkIterator::new_with_range(descriptor, 0..)
+    /// Iterate over revealed spks of all keychains.
+    pub fn revealed_spks(&self) -> impl DoubleEndedIterator<Item = (K, u32, &Script)> + Clone {
+        self.keychains.keys().flat_map(|keychain| {
+            self.revealed_keychain_spks(keychain)
+                .map(|(i, spk)| (keychain.clone(), i, spk))
+        })
     }
 
-    /// Convenience method to get [`revealed_spks_of_keychain`] of all keychains.
-    ///
-    /// [`revealed_spks_of_keychain`]: Self::revealed_spks_of_keychain
-    pub fn revealed_spks_of_all_keychains(
-        &self,
-    ) -> BTreeMap<K, impl Iterator<Item = (u32, &Script)> + Clone> {
-        self.keychains
-            .keys()
-            .map(|keychain| (keychain.clone(), self.revealed_spks_of_keychain(keychain)))
-            .collect()
-    }
-
-    /// Iterates over the script pubkeys revealed by this index under `keychain`.
-    pub fn revealed_spks_of_keychain(
+    /// Iterate over revealed spks of the given `keychain`.
+    pub fn revealed_keychain_spks(
         &self,
         keychain: &K,
     ) -> impl DoubleEndedIterator<Item = (u32, &Script)> + Clone {
-        let next_index = self.last_revealed.get(keychain).map_or(0, |v| *v + 1);
+        let next_i = self.last_revealed.get(keychain).map_or(0, |&i| i + 1);
         self.inner
             .all_spks()
-            .range((keychain.clone(), u32::MIN)..(keychain.clone(), next_index))
-            .map(|((_, derivation_index), spk)| (*derivation_index, spk))
+            .range((keychain.clone(), u32::MIN)..(keychain.clone(), next_i))
+            .map(|((_, i), spk)| (*i, spk.as_script()))
+    }
+
+    /// Iterate over revealed, but unused, spks of all keychains.
+    pub fn unused_spks(&self) -> impl DoubleEndedIterator<Item = (K, u32, &Script)> + Clone {
+        self.keychains.keys().flat_map(|keychain| {
+            self.unused_keychain_spks(keychain)
+                .map(|(i, spk)| (keychain.clone(), i, spk))
+        })
+    }
+
+    /// Iterate over revealed, but unused, spks of the given `keychain`.
+    pub fn unused_keychain_spks(
+        &self,
+        keychain: &K,
+    ) -> impl DoubleEndedIterator<Item = (u32, &Script)> + Clone {
+        let next_i = self.last_revealed.get(keychain).map_or(0, |&i| i + 1);
+        self.inner
+            .unused_spks((keychain.clone(), u32::MIN)..(keychain.clone(), next_i))
+            .map(|((_, i), spk)| (*i, spk))
     }
 
     /// Get the next derivation index for `keychain`. The next index is the index after the last revealed
@@ -370,20 +483,20 @@ impl<K: Clone + Ord + Debug> KeychainTxOutIndex<K> {
         keychains: &BTreeMap<K, u32>,
     ) -> (
         BTreeMap<K, SpkIterator<Descriptor<DescriptorPublicKey>>>,
-        DerivationAdditions<K>,
+        super::ChangeSet<K>,
     ) {
-        let mut additions = DerivationAdditions::default();
+        let mut changeset = super::ChangeSet::default();
         let mut spks = BTreeMap::new();
 
         for (keychain, &index) in keychains {
-            let (new_spks, new_additions) = self.reveal_to_target(keychain, index);
-            if !new_additions.is_empty() {
+            let (new_spks, new_changeset) = self.reveal_to_target(keychain, index);
+            if !new_changeset.is_empty() {
                 spks.insert(keychain.clone(), new_spks);
-                additions.append(new_additions.clone());
+                changeset.append(new_changeset.clone());
             }
         }
 
-        (spks, additions)
+        (spks, changeset)
     }
 
     /// Reveals script pubkeys of the `keychain`'s descriptor **up to and including** the
@@ -394,7 +507,7 @@ impl<K: Clone + Ord + Debug> KeychainTxOutIndex<K> {
     /// reveal up to the last possible index.
     ///
     /// This returns an iterator of newly revealed indices (alongside their scripts) and a
-    /// [`DerivationAdditions`], which reports updates to the latest revealed index. If no new script
+    /// [`super::ChangeSet`], which reports updates to the latest revealed index. If no new script
     /// pubkeys are revealed, then both of these will be empty.
     ///
     /// # Panics
@@ -406,70 +519,60 @@ impl<K: Clone + Ord + Debug> KeychainTxOutIndex<K> {
         target_index: u32,
     ) -> (
         SpkIterator<Descriptor<DescriptorPublicKey>>,
-        DerivationAdditions<K>,
+        super::ChangeSet<K>,
     ) {
         let descriptor = self.keychains.get(keychain).expect("keychain must exist");
         let has_wildcard = descriptor.has_wildcard();
 
         let target_index = if has_wildcard { target_index } else { 0 };
-        let next_reveal_index = self.last_revealed.get(keychain).map_or(0, |v| *v + 1);
-        let lookahead = self.lookahead.get(keychain).map_or(0, |v| *v);
+        let next_reveal_index = self
+            .last_revealed
+            .get(keychain)
+            .map_or(0, |index| *index + 1);
 
-        debug_assert_eq!(
-            next_reveal_index + lookahead,
-            self.next_store_index(keychain)
-        );
+        debug_assert!(next_reveal_index + self.lookahead >= self.next_store_index(keychain));
 
-        // if we need to reveal new indices, the latest revealed index goes here
-        let mut reveal_to_index = None;
-
-        // if the target is not yet revealed, but is already stored (due to lookahead), we need to
-        // set the `reveal_to_index` as target here (as the `for` loop below only updates
-        // `reveal_to_index` for indexes that are NOT stored)
-        if next_reveal_index <= target_index && target_index < next_reveal_index + lookahead {
-            reveal_to_index = Some(target_index);
-        }
-
-        // we range over indexes that are not stored
-        let range = next_reveal_index + lookahead..=target_index + lookahead;
-        for (new_index, new_spk) in SpkIterator::new_with_range(descriptor, range) {
-            let _inserted = self
-                .inner
-                .insert_spk((keychain.clone(), new_index), new_spk);
-            debug_assert!(_inserted, "must not have existing spk",);
-
-            // everything after `target_index` is stored for lookahead only
-            if new_index <= target_index {
-                reveal_to_index = Some(new_index);
-            }
-        }
-
-        match reveal_to_index {
-            Some(index) => {
-                let _old_index = self.last_revealed.insert(keychain.clone(), index);
-                debug_assert!(_old_index < Some(index));
-                (
-                    SpkIterator::new_with_range(descriptor.clone(), next_reveal_index..index + 1),
-                    DerivationAdditions(core::iter::once((keychain.clone(), index)).collect()),
-                )
-            }
-            None => (
+        // If the target_index is already revealed, we are done
+        if next_reveal_index > target_index {
+            return (
                 SpkIterator::new_with_range(
                     descriptor.clone(),
                     next_reveal_index..next_reveal_index,
                 ),
-                DerivationAdditions::default(),
-            ),
+                super::ChangeSet::default(),
+            );
         }
+
+        // We range over the indexes that are not stored and insert their spks in the index.
+        // Indexes from next_reveal_index to next_reveal_index + lookahead are already stored (due
+        // to lookahead), so we only range from next_reveal_index + lookahead to target + lookahead
+        let range = next_reveal_index + self.lookahead..=target_index + self.lookahead;
+        for (new_index, new_spk) in SpkIterator::new_with_range(descriptor, range) {
+            let _inserted = self
+                .inner
+                .insert_spk((keychain.clone(), new_index), new_spk);
+            debug_assert!(_inserted, "must not have existing spk");
+            debug_assert!(
+                has_wildcard || new_index == 0,
+                "non-wildcard descriptors must not iterate past index 0"
+            );
+        }
+
+        let _old_index = self.last_revealed.insert(keychain.clone(), target_index);
+        debug_assert!(_old_index < Some(target_index));
+        (
+            SpkIterator::new_with_range(descriptor.clone(), next_reveal_index..target_index + 1),
+            super::ChangeSet(core::iter::once((keychain.clone(), target_index)).collect()),
+        )
     }
 
     /// Attempts to reveal the next script pubkey for `keychain`.
     ///
     /// Returns the derivation index of the revealed script pubkey, the revealed script pubkey and a
-    /// [`DerivationAdditions`] which represents changes in the last revealed index (if any).
+    /// [`super::ChangeSet`] which represents changes in the last revealed index (if any).
     ///
     /// When a new script cannot be revealed, we return the last revealed script and an empty
-    /// [`DerivationAdditions`]. There are two scenarios when a new script pubkey cannot be derived:
+    /// [`super::ChangeSet`]. There are two scenarios when a new script pubkey cannot be derived:
     ///
     ///  1. The descriptor has no wildcard and already has one script revealed.
     ///  2. The descriptor has already revealed scripts up to the numeric bound.
@@ -477,14 +580,14 @@ impl<K: Clone + Ord + Debug> KeychainTxOutIndex<K> {
     /// # Panics
     ///
     /// Panics if the `keychain` does not exist.
-    pub fn reveal_next_spk(&mut self, keychain: &K) -> ((u32, &Script), DerivationAdditions<K>) {
+    pub fn reveal_next_spk(&mut self, keychain: &K) -> ((u32, &Script), super::ChangeSet<K>) {
         let (next_index, _) = self.next_index(keychain);
-        let additions = self.reveal_to_target(keychain, next_index).1;
+        let changeset = self.reveal_to_target(keychain, next_index).1;
         let script = self
             .inner
             .spk_at_index(&(keychain.clone(), next_index))
             .expect("script must already be stored");
-        ((next_index, script), additions)
+        ((next_index, script), changeset)
     }
 
     /// Gets the next unused script pubkey in the keychain. I.e., the script pubkey with the lowest
@@ -499,73 +602,59 @@ impl<K: Clone + Ord + Debug> KeychainTxOutIndex<K> {
     /// # Panics
     ///
     /// Panics if `keychain` has never been added to the index
-    pub fn next_unused_spk(&mut self, keychain: &K) -> ((u32, &Script), DerivationAdditions<K>) {
-        let need_new = self.unused_spks_of_keychain(keychain).next().is_none();
+    pub fn next_unused_spk(&mut self, keychain: &K) -> ((u32, &Script), super::ChangeSet<K>) {
+        let need_new = self.unused_keychain_spks(keychain).next().is_none();
         // this rather strange branch is needed because of some lifetime issues
         if need_new {
             self.reveal_next_spk(keychain)
         } else {
             (
-                self.unused_spks_of_keychain(keychain)
+                self.unused_keychain_spks(keychain)
                     .next()
                     .expect("we already know next exists"),
-                DerivationAdditions::default(),
+                super::ChangeSet::default(),
             )
         }
     }
 
-    /// Marks the script pubkey at `index` as used even though the tracker hasn't seen an output with it.
-    /// This only has an effect when the `index` had been added to `self` already and was unused.
-    ///
-    /// Returns whether the `index` was initially present as `unused`.
-    ///
-    /// This is useful when you want to reserve a script pubkey for something but don't want to add
-    /// the transaction output using it to the index yet. Other callers will consider `index` on
-    /// `keychain` used until you call [`unmark_used`].
-    ///
-    /// [`unmark_used`]: Self::unmark_used
-    pub fn mark_used(&mut self, keychain: &K, index: u32) -> bool {
-        self.inner.mark_used(&(keychain.clone(), index))
-    }
-
-    /// Undoes the effect of [`mark_used`]. Returns whether the `index` is inserted back into
-    /// `unused`.
-    ///
-    /// Note that if `self` has scanned an output with this script pubkey, then this will have no
-    /// effect.
-    ///
-    /// [`mark_used`]: Self::mark_used
-    pub fn unmark_used(&mut self, keychain: &K, index: u32) -> bool {
-        self.inner.unmark_used(&(keychain.clone(), index))
-    }
-
-    /// Iterates over all unused script pubkeys for a `keychain` stored in the index.
-    pub fn unused_spks_of_keychain(
-        &self,
-        keychain: &K,
-    ) -> impl DoubleEndedIterator<Item = (u32, &Script)> {
-        let next_index = self.last_revealed.get(keychain).map_or(0, |&v| v + 1);
-        let range = (keychain.clone(), u32::MIN)..(keychain.clone(), next_index);
-        self.inner
-            .unused_spks(range)
-            .map(|((_, i), script)| (*i, script))
-    }
-
-    /// Iterates over all the [`OutPoint`] that have a `TxOut` with a script pubkey derived from
+    /// Iterate over all [`OutPoint`]s that point to `TxOut`s with script pubkeys derived from
     /// `keychain`.
-    pub fn txouts_of_keychain(
+    ///
+    /// Use [`keychain_outpoints_in_range`](KeychainTxOutIndex::keychain_outpoints_in_range) to
+    /// iterate over a specific derivation range.
+    pub fn keychain_outpoints(
         &self,
         keychain: &K,
     ) -> impl DoubleEndedIterator<Item = (u32, OutPoint)> + '_ {
+        self.keychain_outpoints_in_range(keychain, ..)
+    }
+
+    /// Iterate over [`OutPoint`]s that point to `TxOut`s with script pubkeys derived from
+    /// `keychain` in a given derivation `range`.
+    pub fn keychain_outpoints_in_range(
+        &self,
+        keychain: &K,
+        range: impl RangeBounds<u32>,
+    ) -> impl DoubleEndedIterator<Item = (u32, OutPoint)> + '_ {
+        let start = match range.start_bound() {
+            Bound::Included(i) => Bound::Included((keychain.clone(), *i)),
+            Bound::Excluded(i) => Bound::Excluded((keychain.clone(), *i)),
+            Bound::Unbounded => Bound::Unbounded,
+        };
+        let end = match range.end_bound() {
+            Bound::Included(i) => Bound::Included((keychain.clone(), *i)),
+            Bound::Excluded(i) => Bound::Excluded((keychain.clone(), *i)),
+            Bound::Unbounded => Bound::Unbounded,
+        };
         self.inner
-            .outputs_in_range((keychain.clone(), u32::MIN)..(keychain.clone(), u32::MAX))
+            .outputs_in_range((start, end))
             .map(|((_, i), op)| (*i, op))
     }
 
     /// Returns the highest derivation index of the `keychain` where [`KeychainTxOutIndex`] has
     /// found a [`TxOut`] with it's script pubkey.
     pub fn last_used_index(&self, keychain: &K) -> Option<u32> {
-        self.txouts_of_keychain(keychain).last().map(|(i, _)| i)
+        self.keychain_outpoints(keychain).last().map(|(i, _)| i)
     }
 
     /// Returns the highest derivation index of each keychain that [`KeychainTxOutIndex`] has found
@@ -580,9 +669,9 @@ impl<K: Clone + Ord + Debug> KeychainTxOutIndex<K> {
             .collect()
     }
 
-    /// Applies the derivation additions to the [`KeychainTxOutIndex`], extending the number of
-    /// derived scripts per keychain, as specified in the `additions`.
-    pub fn apply_additions(&mut self, additions: DerivationAdditions<K>) {
-        let _ = self.reveal_to_target_multi(&additions.0);
+    /// Applies the derivation changeset to the [`KeychainTxOutIndex`], extending the number of
+    /// derived scripts per keychain, as specified in the `changeset`.
+    pub fn apply_changeset(&mut self, changeset: super::ChangeSet<K>) {
+        let _ = self.reveal_to_target_multi(&changeset.0);
     }
 }

crates/chain/src/lib.rs

@@ -1,4 +1,4 @@
-//! This crate is a collection of core structures for [Bitcoin Dev Kit] (alpha release).
+//! This crate is a collection of core structures for [Bitcoin Dev Kit].
 //!
 //! The goal of this crate is to give wallets the mechanisms needed to:
 //!
@@ -12,9 +12,8 @@
 //!    you do it synchronously or asynchronously. If you know a fact about the blockchain, you can just
 //!    tell `bdk_chain`'s APIs about it, and that information will be integrated, if it can be done
 //!    consistently.
-//! 2. Error-free APIs.
-//! 3. Data persistence agnostic -- `bdk_chain` does not care where you cache on-chain data, what you
-//!    cache or how you fetch it.
+//! 2. Data persistence agnostic -- `bdk_chain` does not care where you cache on-chain data, what you
+//!    cache or how you retrieve it from persistent storage.
 //!
 //! [Bitcoin Dev Kit]: https://bitcoindevkit.org/
 

crates/chain/src/local_chain.rs

@@ -2,15 +2,222 @@
 
 use core::convert::Infallible;
 
-use alloc::collections::BTreeMap;
+use crate::collections::BTreeMap;
+use crate::{BlockId, ChainOracle};
+use alloc::sync::Arc;
+use bitcoin::block::Header;
 use bitcoin::BlockHash;
 
-use crate::{BlockId, ChainOracle};
+/// The [`ChangeSet`] represents changes to [`LocalChain`].
+///
+/// The key represents the block height, and the value either represents added a new [`CheckPoint`]
+/// (if [`Some`]), or removing a [`CheckPoint`] (if [`None`]).
+pub type ChangeSet = BTreeMap<u32, Option<BlockHash>>;
+
+/// A [`LocalChain`] checkpoint is used to find the agreement point between two chains and as a
+/// transaction anchor.
+///
+/// Each checkpoint contains the height and hash of a block ([`BlockId`]).
+///
+/// Internally, checkpoints are nodes of a reference-counted linked-list. This allows the caller to
+/// cheaply clone a [`CheckPoint`] without copying the whole list and to view the entire chain
+/// without holding a lock on [`LocalChain`].
+#[derive(Debug, Clone)]
+pub struct CheckPoint(Arc<CPInner>);
+
+/// The internal contents of [`CheckPoint`].
+#[derive(Debug, Clone)]
+struct CPInner {
+    /// Block id (hash and height).
+    block: BlockId,
+    /// Previous checkpoint (if any).
+    prev: Option<Arc<CPInner>>,
+}
+
+impl CheckPoint {
+    /// Construct a new base block at the front of a linked list.
+    pub fn new(block: BlockId) -> Self {
+        Self(Arc::new(CPInner { block, prev: None }))
+    }
+
+    /// Construct a checkpoint from a list of [`BlockId`]s in ascending height order.
+    ///
+    /// # Errors
+    ///
+    /// This method will error if any of the follow occurs:
+    ///
+    /// - The `blocks` iterator is empty, in which case, the error will be `None`.
+    /// - The `blocks` iterator is not in ascending height order.
+    /// - The `blocks` iterator contains multiple [`BlockId`]s of the same height.
+    ///
+    /// The error type is the last successful checkpoint constructed (if any).
+    pub fn from_block_ids(
+        block_ids: impl IntoIterator<Item = BlockId>,
+    ) -> Result<Self, Option<Self>> {
+        let mut blocks = block_ids.into_iter();
+        let mut acc = CheckPoint::new(blocks.next().ok_or(None)?);
+        for id in blocks {
+            acc = acc.push(id).map_err(Some)?;
+        }
+        Ok(acc)
+    }
+
+    /// Construct a checkpoint from the given `header` and block `height`.
+    ///
+    /// If `header` is of the genesis block, the checkpoint won't have a [`prev`] node. Otherwise,
+    /// we return a checkpoint linked with the previous block.
+    ///
+    /// [`prev`]: CheckPoint::prev
+    pub fn from_header(header: &bitcoin::block::Header, height: u32) -> Self {
+        let hash = header.block_hash();
+        let this_block_id = BlockId { height, hash };
+
+        let prev_height = match height.checked_sub(1) {
+            Some(h) => h,
+            None => return Self::new(this_block_id),
+        };
+
+        let prev_block_id = BlockId {
+            height: prev_height,
+            hash: header.prev_blockhash,
+        };
+
+        CheckPoint::new(prev_block_id)
+            .push(this_block_id)
+            .expect("must construct checkpoint")
+    }
+
+    /// Convenience method to convert the [`CheckPoint`] into an [`Update`].
+    ///
+    /// For more information, refer to [`Update`].
+    pub fn into_update(self, introduce_older_blocks: bool) -> Update {
+        Update {
+            tip: self,
+            introduce_older_blocks,
+        }
+    }
+
+    /// Puts another checkpoint onto the linked list representing the blockchain.
+    ///
+    /// Returns an `Err(self)` if the block you are pushing on is not at a greater height that the one you
+    /// are pushing on to.
+    pub fn push(self, block: BlockId) -> Result<Self, Self> {
+        if self.height() < block.height {
+            Ok(Self(Arc::new(CPInner {
+                block,
+                prev: Some(self.0),
+            })))
+        } else {
+            Err(self)
+        }
+    }
+
+    /// Extends the checkpoint linked list by a iterator of block ids.
+    ///
+    /// Returns an `Err(self)` if there is block which does not have a greater height than the
+    /// previous one.
+    pub fn extend(self, blocks: impl IntoIterator<Item = BlockId>) -> Result<Self, Self> {
+        let mut curr = self.clone();
+        for block in blocks {
+            curr = curr.push(block).map_err(|_| self.clone())?;
+        }
+        Ok(curr)
+    }
+
+    /// Get the [`BlockId`] of the checkpoint.
+    pub fn block_id(&self) -> BlockId {
+        self.0.block
+    }
+
+    /// Get the height of the checkpoint.
+    pub fn height(&self) -> u32 {
+        self.0.block.height
+    }
+
+    /// Get the block hash of the checkpoint.
+    pub fn hash(&self) -> BlockHash {
+        self.0.block.hash
+    }
+
+    /// Get the previous checkpoint in the chain
+    pub fn prev(&self) -> Option<CheckPoint> {
+        self.0.prev.clone().map(CheckPoint)
+    }
+
+    /// Iterate from this checkpoint in descending height.
+    pub fn iter(&self) -> CheckPointIter {
+        self.clone().into_iter()
+    }
+}
+
+/// Iterates over checkpoints backwards.
+pub struct CheckPointIter {
+    current: Option<Arc<CPInner>>,
+}
+
+impl Iterator for CheckPointIter {
+    type Item = CheckPoint;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        let current = self.current.clone()?;
+        self.current = current.prev.clone();
+        Some(CheckPoint(current))
+    }
+}
+
+impl IntoIterator for CheckPoint {
+    type Item = CheckPoint;
+    type IntoIter = CheckPointIter;
+
+    fn into_iter(self) -> Self::IntoIter {
+        CheckPointIter {
+            current: Some(self.0),
+        }
+    }
+}
+
+/// Used to update [`LocalChain`].
+///
+/// This is used as input for [`LocalChain::apply_update`]. It contains the update's chain `tip` and
+/// a flag `introduce_older_blocks` which signals whether this update intends to introduce missing
+/// blocks to the original chain.
+///
+/// Block-by-block syncing mechanisms would typically create updates that builds upon the previous
+/// tip. In this case, `introduce_older_blocks` would be `false`.
+///
+/// Script-pubkey based syncing mechanisms may not introduce transactions in a chronological order
+/// so some updates require introducing older blocks (to anchor older transactions). For
+/// script-pubkey based syncing, `introduce_older_blocks` would typically be `true`.
+#[derive(Debug, Clone)]
+pub struct Update {
+    /// The update chain's new tip.
+    pub tip: CheckPoint,
+
+    /// Whether the update allows for introducing older blocks.
+    ///
+    /// Refer to [struct-level documentation] for more.
+    ///
+    /// [struct-level documentation]: Update
+    pub introduce_older_blocks: bool,
+}
 
 /// This is a local implementation of [`ChainOracle`].
-#[derive(Debug, Default, Clone, PartialEq, Eq, PartialOrd, Ord)]
+#[derive(Debug, Clone)]
 pub struct LocalChain {
-    blocks: BTreeMap<u32, BlockHash>,
+    tip: CheckPoint,
+    index: BTreeMap<u32, BlockHash>,
+}
+
+impl PartialEq for LocalChain {
+    fn eq(&self, other: &Self) -> bool {
+        self.index == other.index
+    }
+}
+
+impl From<LocalChain> for BTreeMap<u32, BlockHash> {
+    fn from(value: LocalChain) -> Self {
+        value.index
+    }
 }
 
 impl ChainOracle for LocalChain {
@@ -19,232 +226,574 @@ impl ChainOracle for LocalChain {
     fn is_block_in_chain(
         &self,
         block: BlockId,
-        static_block: BlockId,
+        chain_tip: BlockId,
     ) -> Result<Option<bool>, Self::Error> {
-        if block.height > static_block.height {
+        if block.height > chain_tip.height {
             return Ok(None);
         }
         Ok(
             match (
-                self.blocks.get(&block.height),
-                self.blocks.get(&static_block.height),
+                self.index.get(&block.height),
+                self.index.get(&chain_tip.height),
             ) {
-                (Some(&hash), Some(&static_hash)) => {
-                    Some(hash == block.hash && static_hash == static_block.hash)
-                }
+                (Some(cp), Some(tip_cp)) => Some(*cp == block.hash && *tip_cp == chain_tip.hash),
                 _ => None,
             },
         )
     }
 
-    fn get_chain_tip(&self) -> Result<Option<BlockId>, Self::Error> {
-        Ok(self.tip())
-    }
-}
-
-impl AsRef<BTreeMap<u32, BlockHash>> for LocalChain {
-    fn as_ref(&self) -> &BTreeMap<u32, BlockHash> {
-        &self.blocks
-    }
-}
-
-impl From<LocalChain> for BTreeMap<u32, BlockHash> {
-    fn from(value: LocalChain) -> Self {
-        value.blocks
-    }
-}
-
-impl From<BTreeMap<u32, BlockHash>> for LocalChain {
-    fn from(value: BTreeMap<u32, BlockHash>) -> Self {
-        Self { blocks: value }
+    fn get_chain_tip(&self) -> Result<BlockId, Self::Error> {
+        Ok(self.tip.block_id())
     }
 }
 
 impl LocalChain {
-    /// Contruct a [`LocalChain`] from a list of [`BlockId`]s.
-    pub fn from_blocks<B>(blocks: B) -> Self
-    where
-        B: IntoIterator<Item = BlockId>,
-    {
-        Self {
-            blocks: blocks.into_iter().map(|b| (b.height, b.hash)).collect(),
-        }
+    /// Get the genesis hash.
+    pub fn genesis_hash(&self) -> BlockHash {
+        self.index.get(&0).copied().expect("must have genesis hash")
     }
 
-    /// Get a reference to a map of block height to hash.
-    pub fn blocks(&self) -> &BTreeMap<u32, BlockHash> {
-        &self.blocks
+    /// Construct [`LocalChain`] from genesis `hash`.
+    #[must_use]
+    pub fn from_genesis_hash(hash: BlockHash) -> (Self, ChangeSet) {
+        let height = 0;
+        let chain = Self {
+            tip: CheckPoint::new(BlockId { height, hash }),
+            index: core::iter::once((height, hash)).collect(),
+        };
+        let changeset = chain.initial_changeset();
+        (chain, changeset)
     }
 
-    /// Get the chain tip.
-    pub fn tip(&self) -> Option<BlockId> {
-        self.blocks
-            .iter()
-            .last()
-            .map(|(&height, &hash)| BlockId { height, hash })
-    }
-
-    /// This is like the sparsechain's logic, expect we must guarantee that all invalidated heights
-    /// are to be re-filled.
-    pub fn determine_changeset(&self, update: &Self) -> Result<ChangeSet, UpdateNotConnectedError> {
-        let update = update.as_ref();
-        let update_tip = match update.keys().last().cloned() {
-            Some(tip) => tip,
-            None => return Ok(ChangeSet::default()),
+    /// Construct a [`LocalChain`] from an initial `changeset`.
+    pub fn from_changeset(changeset: ChangeSet) -> Result<Self, MissingGenesisError> {
+        let genesis_entry = changeset.get(&0).copied().flatten();
+        let genesis_hash = match genesis_entry {
+            Some(hash) => hash,
+            None => return Err(MissingGenesisError),
         };
 
-        // this is the latest height where both the update and local chain has the same block hash
-        let agreement_height = update
-            .iter()
-            .rev()
-            .find(|&(u_height, u_hash)| self.blocks.get(u_height) == Some(u_hash))
-            .map(|(&height, _)| height);
+        let (mut chain, _) = Self::from_genesis_hash(genesis_hash);
+        chain.apply_changeset(&changeset)?;
 
-        // the lower bound of the range to invalidate
-        let invalidate_lb = match agreement_height {
-            Some(height) if height == update_tip => u32::MAX,
-            Some(height) => height + 1,
-            None => 0,
+        debug_assert!(chain._check_index_is_consistent_with_tip());
+        debug_assert!(chain._check_changeset_is_applied(&changeset));
+
+        Ok(chain)
+    }
+
+    /// Construct a [`LocalChain`] from a given `checkpoint` tip.
+    pub fn from_tip(tip: CheckPoint) -> Result<Self, MissingGenesisError> {
+        let mut chain = Self {
+            tip,
+            index: BTreeMap::new(),
         };
+        chain.reindex(0);
 
-        // the first block's height to invalidate in the local chain
-        let invalidate_from_height = self.blocks.range(invalidate_lb..).next().map(|(&h, _)| h);
-
-        // the first block of height to invalidate (if any) should be represented in the update
-        if let Some(first_invalid_height) = invalidate_from_height {
-            if !update.contains_key(&first_invalid_height) {
-                return Err(UpdateNotConnectedError(first_invalid_height));
-            }
+        if chain.index.get(&0).copied().is_none() {
+            return Err(MissingGenesisError);
         }
 
-        let mut changeset: BTreeMap<u32, Option<BlockHash>> = match invalidate_from_height {
-            Some(first_invalid_height) => {
-                // the first block of height to invalidate should be represented in the update
-                if !update.contains_key(&first_invalid_height) {
-                    return Err(UpdateNotConnectedError(first_invalid_height));
+        debug_assert!(chain._check_index_is_consistent_with_tip());
+        Ok(chain)
+    }
+
+    /// Constructs a [`LocalChain`] from a [`BTreeMap`] of height to [`BlockHash`].
+    ///
+    /// The [`BTreeMap`] enforces the height order. However, the caller must ensure the blocks are
+    /// all of the same chain.
+    pub fn from_blocks(blocks: BTreeMap<u32, BlockHash>) -> Result<Self, MissingGenesisError> {
+        if !blocks.contains_key(&0) {
+            return Err(MissingGenesisError);
+        }
+
+        let mut tip: Option<CheckPoint> = None;
+
+        for block in &blocks {
+            match tip {
+                Some(curr) => {
+                    tip = Some(
+                        curr.push(BlockId::from(block))
+                            .expect("BTreeMap is ordered"),
+                    )
                 }
-                self.blocks
-                    .range(first_invalid_height..)
-                    .map(|(height, _)| (*height, None))
-                    .collect()
+                None => tip = Some(CheckPoint::new(BlockId::from(block))),
             }
-            None => BTreeMap::new(),
+        }
+
+        let chain = Self {
+            index: blocks,
+            tip: tip.expect("already checked to have genesis"),
         };
-        for (height, update_hash) in update {
-            let original_hash = self.blocks.get(height);
-            if Some(update_hash) != original_hash {
-                changeset.insert(*height, Some(*update_hash));
-            }
-        }
 
+        debug_assert!(chain._check_index_is_consistent_with_tip());
+        Ok(chain)
+    }
+
+    /// Get the highest checkpoint.
+    pub fn tip(&self) -> CheckPoint {
+        self.tip.clone()
+    }
+
+    /// Applies the given `update` to the chain.
+    ///
+    /// The method returns [`ChangeSet`] on success. This represents the applied changes to `self`.
+    ///
+    /// There must be no ambiguity about which of the existing chain's blocks are still valid and
+    /// which are now invalid. That is, the new chain must implicitly connect to a definite block in
+    /// the existing chain and invalidate the block after it (if it exists) by including a block at
+    /// the same height but with a different hash to explicitly exclude it as a connection point.
+    ///
+    /// Additionally, an empty chain can be updated with any chain, and a chain with a single block
+    /// can have it's block invalidated by an update chain with a block at the same height but
+    /// different hash.
+    ///
+    /// # Errors
+    ///
+    /// An error will occur if the update does not correctly connect with `self`.
+    ///
+    /// Refer to [`Update`] for more about the update struct.
+    ///
+    /// [module-level documentation]: crate::local_chain
+    pub fn apply_update(&mut self, update: Update) -> Result<ChangeSet, CannotConnectError> {
+        let changeset = merge_chains(
+            self.tip.clone(),
+            update.tip.clone(),
+            update.introduce_older_blocks,
+        )?;
+        // `._check_index_is_consistent_with_tip` and `._check_changeset_is_applied` is called in
+        // `.apply_changeset`
+        self.apply_changeset(&changeset)
+            .map_err(|_| CannotConnectError {
+                try_include_height: 0,
+            })?;
         Ok(changeset)
     }
 
-    /// Applies the given `changeset`.
-    pub fn apply_changeset(&mut self, changeset: ChangeSet) {
-        for (height, blockhash) in changeset {
-            match blockhash {
-                Some(blockhash) => self.blocks.insert(height, blockhash),
-                None => self.blocks.remove(&height),
-            };
-        }
-    }
-
-    /// Updates [`LocalChain`] with an update [`LocalChain`].
+    /// Update the chain with a given [`Header`] at `height` which you claim is connected to a existing block in the chain.
     ///
-    /// This is equivalent to calling [`determine_changeset`] and [`apply_changeset`] in sequence.
+    /// This is useful when you have a block header that you want to record as part of the chain but
+    /// don't necessarily know that the `prev_blockhash` is in the chain.
     ///
-    /// [`determine_changeset`]: Self::determine_changeset
-    /// [`apply_changeset`]: Self::apply_changeset
-    pub fn apply_update(&mut self, update: Self) -> Result<ChangeSet, UpdateNotConnectedError> {
-        let changeset = self.determine_changeset(&update)?;
-        self.apply_changeset(changeset.clone());
-        Ok(changeset)
-    }
-
-    /// Derives a [`ChangeSet`] that assumes that there are no preceding changesets.
+    /// This will usually insert two new [`BlockId`]s into the chain: the header's block and the
+    /// header's `prev_blockhash` block. `connected_to` must already be in the chain but is allowed
+    /// to be `prev_blockhash` (in which case only one new block id will be inserted).
+    /// To be successful, `connected_to` must be chosen carefully so that `LocalChain`'s [update
+    /// rules][`apply_update`] are satisfied.
     ///
-    /// The changeset returned will record additions of all blocks included in [`Self`].
-    pub fn initial_changeset(&self) -> ChangeSet {
-        self.blocks
-            .iter()
-            .map(|(&height, &hash)| (height, Some(hash)))
-            .collect()
-    }
-
-    /// Insert a block of [`BlockId`] into the [`LocalChain`].
+    /// # Errors
     ///
-    /// # Error
+    /// [`ApplyHeaderError::InconsistentBlocks`] occurs if the `connected_to` block and the
+    /// [`Header`] is inconsistent. For example, if the `connected_to` block is the same height as
+    /// `header` or `prev_blockhash`, but has a different block hash. Or if the `connected_to`
+    /// height is greater than the header's `height`.
     ///
-    /// If the insertion height already contains a block, and the block has a different blockhash,
-    /// this will result in an [`InsertBlockNotMatchingError`].
-    pub fn insert_block(
+    /// [`ApplyHeaderError::CannotConnect`] occurs if the internal call to [`apply_update`] fails.
+    ///
+    /// [`apply_update`]: Self::apply_update
+    pub fn apply_header_connected_to(
         &mut self,
-        block_id: BlockId,
-    ) -> Result<ChangeSet, InsertBlockNotMatchingError> {
-        let mut update = Self::from_blocks(self.tip());
+        header: &Header,
+        height: u32,
+        connected_to: BlockId,
+    ) -> Result<ChangeSet, ApplyHeaderError> {
+        let this = BlockId {
+            height,
+            hash: header.block_hash(),
+        };
+        let prev = height.checked_sub(1).map(|prev_height| BlockId {
+            height: prev_height,
+            hash: header.prev_blockhash,
+        });
+        let conn = match connected_to {
+            // `connected_to` can be ignored if same as `this` or `prev` (duplicate)
+            conn if conn == this || Some(conn) == prev => None,
+            // this occurs if:
+            // - `connected_to` height is the same as `prev`, but different hash
+            // - `connected_to` height is the same as `this`, but different hash
+            // - `connected_to` height is greater than `this` (this is not allowed)
+            conn if conn.height >= height.saturating_sub(1) => {
+                return Err(ApplyHeaderError::InconsistentBlocks)
+            }
+            conn => Some(conn),
+        };
 
-        if let Some(original_hash) = update.blocks.insert(block_id.height, block_id.hash) {
+        let update = Update {
+            tip: CheckPoint::from_block_ids([conn, prev, Some(this)].into_iter().flatten())
+                .expect("block ids must be in order"),
+            introduce_older_blocks: false,
+        };
+
+        self.apply_update(update)
+            .map_err(ApplyHeaderError::CannotConnect)
+    }
+
+    /// Update the chain with a given [`Header`] connecting it with the previous block.
+    ///
+    /// This is a convenience method to call [`apply_header_connected_to`] with the `connected_to`
+    /// parameter being `height-1:prev_blockhash`. If there is no previous block (i.e. genesis), we
+    /// use the current block as `connected_to`.
+    ///
+    /// [`apply_header_connected_to`]: LocalChain::apply_header_connected_to
+    pub fn apply_header(
+        &mut self,
+        header: &Header,
+        height: u32,
+    ) -> Result<ChangeSet, CannotConnectError> {
+        let connected_to = match height.checked_sub(1) {
+            Some(prev_height) => BlockId {
+                height: prev_height,
+                hash: header.prev_blockhash,
+            },
+            None => BlockId {
+                height,
+                hash: header.block_hash(),
+            },
+        };
+        self.apply_header_connected_to(header, height, connected_to)
+            .map_err(|err| match err {
+                ApplyHeaderError::InconsistentBlocks => {
+                    unreachable!("connected_to is derived from the block so is always consistent")
+                }
+                ApplyHeaderError::CannotConnect(err) => err,
+            })
+    }
+
+    /// Apply the given `changeset`.
+    pub fn apply_changeset(&mut self, changeset: &ChangeSet) -> Result<(), MissingGenesisError> {
+        if let Some(start_height) = changeset.keys().next().cloned() {
+            // changes after point of agreement
+            let mut extension = BTreeMap::default();
+            // point of agreement
+            let mut base: Option<CheckPoint> = None;
+
+            for cp in self.iter_checkpoints() {
+                if cp.height() >= start_height {
+                    extension.insert(cp.height(), cp.hash());
+                } else {
+                    base = Some(cp);
+                    break;
+                }
+            }
+
+            for (&height, &hash) in changeset {
+                match hash {
+                    Some(hash) => {
+                        extension.insert(height, hash);
+                    }
+                    None => {
+                        extension.remove(&height);
+                    }
+                };
+            }
+
+            let new_tip = match base {
+                Some(base) => base
+                    .extend(extension.into_iter().map(BlockId::from))
+                    .expect("extension is strictly greater than base"),
+                None => LocalChain::from_blocks(extension)?.tip(),
+            };
+            self.tip = new_tip;
+            self.reindex(start_height);
+
+            debug_assert!(self._check_index_is_consistent_with_tip());
+            debug_assert!(self._check_changeset_is_applied(changeset));
+        }
+
+        Ok(())
+    }
+
+    /// Insert a [`BlockId`].
+    ///
+    /// # Errors
+    ///
+    /// Replacing the block hash of an existing checkpoint will result in an error.
+    pub fn insert_block(&mut self, block_id: BlockId) -> Result<ChangeSet, AlterCheckPointError> {
+        if let Some(&original_hash) = self.index.get(&block_id.height) {
             if original_hash != block_id.hash {
-                return Err(InsertBlockNotMatchingError {
+                return Err(AlterCheckPointError {
                     height: block_id.height,
                     original_hash,
-                    update_hash: block_id.hash,
+                    update_hash: Some(block_id.hash),
                 });
+            } else {
+                return Ok(ChangeSet::default());
             }
         }
 
-        Ok(self.apply_update(update).expect("should always connect"))
+        let mut changeset = ChangeSet::default();
+        changeset.insert(block_id.height, Some(block_id.hash));
+        self.apply_changeset(&changeset)
+            .map_err(|_| AlterCheckPointError {
+                height: 0,
+                original_hash: self.genesis_hash(),
+                update_hash: changeset.get(&0).cloned().flatten(),
+            })?;
+        Ok(changeset)
+    }
+
+    /// Removes blocks from (and inclusive of) the given `block_id`.
+    ///
+    /// This will remove blocks with a height equal or greater than `block_id`, but only if
+    /// `block_id` exists in the chain.
+    ///
+    /// # Errors
+    ///
+    /// This will fail with [`MissingGenesisError`] if the caller attempts to disconnect from the
+    /// genesis block.
+    pub fn disconnect_from(&mut self, block_id: BlockId) -> Result<ChangeSet, MissingGenesisError> {
+        if self.index.get(&block_id.height) != Some(&block_id.hash) {
+            return Ok(ChangeSet::default());
+        }
+
+        let changeset = self
+            .index
+            .range(block_id.height..)
+            .map(|(&height, _)| (height, None))
+            .collect::<ChangeSet>();
+        self.apply_changeset(&changeset).map(|_| changeset)
+    }
+
+    /// Reindex the heights in the chain from (and including) `from` height
+    fn reindex(&mut self, from: u32) {
+        let _ = self.index.split_off(&from);
+        for cp in self.iter_checkpoints() {
+            if cp.height() < from {
+                break;
+            }
+            self.index.insert(cp.height(), cp.hash());
+        }
+    }
+
+    /// Derives an initial [`ChangeSet`], meaning that it can be applied to an empty chain to
+    /// recover the current chain.
+    pub fn initial_changeset(&self) -> ChangeSet {
+        self.index.iter().map(|(k, v)| (*k, Some(*v))).collect()
+    }
+
+    /// Iterate over checkpoints in descending height order.
+    pub fn iter_checkpoints(&self) -> CheckPointIter {
+        CheckPointIter {
+            current: Some(self.tip.0.clone()),
+        }
+    }
+
+    /// Get a reference to the internal index mapping the height to block hash.
+    pub fn blocks(&self) -> &BTreeMap<u32, BlockHash> {
+        &self.index
+    }
+
+    fn _check_index_is_consistent_with_tip(&self) -> bool {
+        let tip_history = self
+            .tip
+            .iter()
+            .map(|cp| (cp.height(), cp.hash()))
+            .collect::<BTreeMap<_, _>>();
+        self.index == tip_history
+    }
+
+    fn _check_changeset_is_applied(&self, changeset: &ChangeSet) -> bool {
+        for (height, exp_hash) in changeset {
+            if self.index.get(height) != exp_hash.as_ref() {
+                return false;
+            }
+        }
+        true
     }
 }
 
-/// This is the return value of [`determine_changeset`] and represents changes to [`LocalChain`].
-///
-/// [`determine_changeset`]: LocalChain::determine_changeset
-pub type ChangeSet = BTreeMap<u32, Option<BlockHash>>;
-
-/// Represents an update failure of [`LocalChain`] due to the update not connecting to the original
-/// chain.
-///
-/// The update cannot be applied to the chain because the chain suffix it represents did not
-/// connect to the existing chain. This error case contains the checkpoint height to include so
-/// that the chains can connect.
+/// An error which occurs when a [`LocalChain`] is constructed without a genesis checkpoint.
 #[derive(Clone, Debug, PartialEq)]
-pub struct UpdateNotConnectedError(pub u32);
+pub struct MissingGenesisError;
 
-impl core::fmt::Display for UpdateNotConnectedError {
+impl core::fmt::Display for MissingGenesisError {
     fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
         write!(
             f,
-            "the update cannot connect with the chain, try include block at height {}",
-            self.0
+            "cannot construct `LocalChain` without a genesis checkpoint"
         )
     }
 }
 
 #[cfg(feature = "std")]
-impl std::error::Error for UpdateNotConnectedError {}
+impl std::error::Error for MissingGenesisError {}
 
-/// Represents a failure when trying to insert a checkpoint into [`LocalChain`].
+/// Represents a failure when trying to insert/remove a checkpoint to/from [`LocalChain`].
 #[derive(Clone, Debug, PartialEq)]
-pub struct InsertBlockNotMatchingError {
-    /// The checkpoints' height.
+pub struct AlterCheckPointError {
+    /// The checkpoint's height.
     pub height: u32,
-    /// Original checkpoint's block hash.
+    /// The original checkpoint's block hash which cannot be replaced/removed.
     pub original_hash: BlockHash,
-    /// Update checkpoint's block hash.
-    pub update_hash: BlockHash,
+    /// The attempted update to the `original_block` hash.
+    pub update_hash: Option<BlockHash>,
 }
 
-impl core::fmt::Display for InsertBlockNotMatchingError {
+impl core::fmt::Display for AlterCheckPointError {
+    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+        match self.update_hash {
+            Some(update_hash) => write!(
+                f,
+                "failed to insert block at height {}: original={} update={}",
+                self.height, self.original_hash, update_hash
+            ),
+            None => write!(
+                f,
+                "failed to remove block at height {}: original={}",
+                self.height, self.original_hash
+            ),
+        }
+    }
+}
+
+#[cfg(feature = "std")]
+impl std::error::Error for AlterCheckPointError {}
+
+/// Occurs when an update does not have a common checkpoint with the original chain.
+#[derive(Clone, Debug, PartialEq)]
+pub struct CannotConnectError {
+    /// The suggested checkpoint to include to connect the two chains.
+    pub try_include_height: u32,
+}
+
+impl core::fmt::Display for CannotConnectError {
     fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
         write!(
             f,
-            "failed to insert block at height {} as blockhashes conflict: original={}, update={}",
-            self.height, self.original_hash, self.update_hash
+            "introduced chain cannot connect with the original chain, try include height {}",
+            self.try_include_height,
         )
     }
 }
 
 #[cfg(feature = "std")]
-impl std::error::Error for InsertBlockNotMatchingError {}
+impl std::error::Error for CannotConnectError {}
+
+/// The error type for [`LocalChain::apply_header_connected_to`].
+#[derive(Debug, Clone, PartialEq)]
+pub enum ApplyHeaderError {
+    /// Occurs when `connected_to` block conflicts with either the current block or previous block.
+    InconsistentBlocks,
+    /// Occurs when the update cannot connect with the original chain.
+    CannotConnect(CannotConnectError),
+}
+
+impl core::fmt::Display for ApplyHeaderError {
+    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+        match self {
+            ApplyHeaderError::InconsistentBlocks => write!(
+                f,
+                "the `connected_to` block conflicts with either the current or previous block"
+            ),
+            ApplyHeaderError::CannotConnect(err) => core::fmt::Display::fmt(err, f),
+        }
+    }
+}
+
+#[cfg(feature = "std")]
+impl std::error::Error for ApplyHeaderError {}
+
+fn merge_chains(
+    original_tip: CheckPoint,
+    update_tip: CheckPoint,
+    introduce_older_blocks: bool,
+) -> Result<ChangeSet, CannotConnectError> {
+    let mut changeset = ChangeSet::default();
+    let mut orig = original_tip.into_iter();
+    let mut update = update_tip.into_iter();
+    let mut curr_orig = None;
+    let mut curr_update = None;
+    let mut prev_orig: Option<CheckPoint> = None;
+    let mut prev_update: Option<CheckPoint> = None;
+    let mut point_of_agreement_found = false;
+    let mut prev_orig_was_invalidated = false;
+    let mut potentially_invalidated_heights = vec![];
+
+    // To find the difference between the new chain and the original we iterate over both of them
+    // from the tip backwards in tandem. We always dealing with the highest one from either chain
+    // first and move to the next highest. The crucial logic is applied when they have blocks at the
+    // same height.
+    loop {
+        if curr_orig.is_none() {
+            curr_orig = orig.next();
+        }
+        if curr_update.is_none() {
+            curr_update = update.next();
+        }
+
+        match (curr_orig.as_ref(), curr_update.as_ref()) {
+            // Update block that doesn't exist in the original chain
+            (o, Some(u)) if Some(u.height()) > o.map(|o| o.height()) => {
+                changeset.insert(u.height(), Some(u.hash()));
+                prev_update = curr_update.take();
+            }
+            // Original block that isn't in the update
+            (Some(o), u) if Some(o.height()) > u.map(|u| u.height()) => {
+                // this block might be gone if an earlier block gets invalidated
+                potentially_invalidated_heights.push(o.height());
+                prev_orig_was_invalidated = false;
+                prev_orig = curr_orig.take();
+
+                // OPTIMIZATION: we have run out of update blocks so we don't need to continue
+                // iterating because there's no possibility of adding anything to changeset.
+                if u.is_none() {
+                    break;
+                }
+            }
+            (Some(o), Some(u)) => {
+                if o.hash() == u.hash() {
+                    // We have found our point of agreement 🎉 -- we require that the previous (i.e.
+                    // higher because we are iterating backwards) block in the original chain was
+                    // invalidated (if it exists). This ensures that there is an unambiguous point of
+                    // connection to the original chain from the update chain (i.e. we know the
+                    // precisely which original blocks are invalid).
+                    if !prev_orig_was_invalidated && !point_of_agreement_found {
+                        if let (Some(prev_orig), Some(_prev_update)) = (&prev_orig, &prev_update) {
+                            return Err(CannotConnectError {
+                                try_include_height: prev_orig.height(),
+                            });
+                        }
+                    }
+                    point_of_agreement_found = true;
+                    prev_orig_was_invalidated = false;
+                    // OPTIMIZATION 1 -- If we know that older blocks cannot be introduced without
+                    // invalidation, we can break after finding the point of agreement.
+                    // OPTIMIZATION 2 -- if we have the same underlying pointer at this point, we
+                    // can guarantee that no older blocks are introduced.
+                    if !introduce_older_blocks || Arc::as_ptr(&o.0) == Arc::as_ptr(&u.0) {
+                        return Ok(changeset);
+                    }
+                } else {
+                    // We have an invalidation height so we set the height to the updated hash and
+                    // also purge all the original chain block hashes above this block.
+                    changeset.insert(u.height(), Some(u.hash()));
+                    for invalidated_height in potentially_invalidated_heights.drain(..) {
+                        changeset.insert(invalidated_height, None);
+                    }
+                    prev_orig_was_invalidated = true;
+                }
+                prev_update = curr_update.take();
+                prev_orig = curr_orig.take();
+            }
+            (None, None) => {
+                break;
+            }
+            _ => {
+                unreachable!("compiler cannot tell that everything has been covered")
+            }
+        }
+    }
+
+    // When we don't have a point of agreement you can imagine it is implicitly the
+    // genesis block so we need to do the final connectivity check which in this case
+    // just means making sure the entire original chain was invalidated.
+    if !prev_orig_was_invalidated && !point_of_agreement_found {
+        if let Some(prev_orig) = prev_orig {
+            return Err(CannotConnectError {
+                try_include_height: prev_orig.height(),
+            });
+        }
+    }
+
+    Ok(changeset)
+}

crates/chain/src/persist.rs

@@ -27,19 +27,19 @@ where
         }
     }
 
-    /// Stage a `changeset` to be commited later with [`commit`].
+    /// Stage a `changeset` to be committed later with [`commit`].
     ///
     /// [`commit`]: Self::commit
     pub fn stage(&mut self, changeset: C) {
         self.stage.append(changeset)
     }
 
-    /// Get the changes that have not been commited yet.
+    /// Get the changes that have not been committed yet.
     pub fn staged(&self) -> &C {
         &self.stage
     }
 
-    /// Commit the staged changes to the underlying persistance backend.
+    /// Commit the staged changes to the underlying persistence backend.
     ///
     /// Changes that are committed (if any) are returned.
     ///
@@ -55,6 +55,18 @@ where
             // if written successfully, take and return `self.stage`
             .map(|_| Some(core::mem::take(&mut self.stage)))
     }
+
+    /// Stages a new changeset and commits it (along with any other previously staged changes) to
+    /// the persistence backend
+    ///
+    /// Convenience method for calling [`stage`] and then [`commit`].
+    ///
+    /// [`stage`]: Self::stage
+    /// [`commit`]: Self::commit
+    pub fn stage_and_commit(&mut self, changeset: C) -> Result<Option<C>, B::WriteError> {
+        self.stage(changeset);
+        self.commit()
+    }
 }
 
 /// A persistence backend for [`Persist`].
@@ -79,10 +91,10 @@ pub trait PersistBackend<C> {
     fn write_changes(&mut self, changeset: &C) -> Result<(), Self::WriteError>;
 
     /// Return the aggregate changeset `C` from persistence.
-    fn load_from_persistence(&mut self) -> Result<C, Self::LoadError>;
+    fn load_from_persistence(&mut self) -> Result<Option<C>, Self::LoadError>;
 }
 
-impl<C: Default> PersistBackend<C> for () {
+impl<C> PersistBackend<C> for () {
     type WriteError = Infallible;
 
     type LoadError = Infallible;
@@ -91,7 +103,7 @@ impl<C: Default> PersistBackend<C> for () {
         Ok(())
     }
 
-    fn load_from_persistence(&mut self) -> Result<C, Self::LoadError> {
-        Ok(C::default())
+    fn load_from_persistence(&mut self) -> Result<Option<C>, Self::LoadError> {
+        Ok(None)
     }
 }

crates/chain/src/spk_iter.rs

@@ -1,5 +1,5 @@
 use crate::{
-    bitcoin::{secp256k1::Secp256k1, Script},
+    bitcoin::{secp256k1::Secp256k1, ScriptBuf},
     miniscript::{Descriptor, DescriptorPublicKey},
 };
 use core::{borrow::Borrow, ops::Bound, ops::RangeBounds};
@@ -22,9 +22,9 @@ pub const BIP32_MAX_INDEX: u32 = (1 << 31) - 1;
 /// # use std::str::FromStr;
 /// # let secp = bitcoin::secp256k1::Secp256k1::signing_only();
 /// # let (descriptor, _) = Descriptor::<DescriptorPublicKey>::parse_descriptor(&secp, "wpkh([73c5da0a/86'/0'/0']xprv9xgqHN7yz9MwCkxsBPN5qetuNdQSUttZNKw1dcYTV4mkaAFiBVGQziHs3NRSWMkCzvgjEe3n9xV8oYywvM8at9yRqyaZVz6TYYhX98VjsUk/1/0)").unwrap();
-/// # let external_spk_0 = descriptor.at_derivation_index(0).script_pubkey();
-/// # let external_spk_3 = descriptor.at_derivation_index(3).script_pubkey();
-/// # let external_spk_4 = descriptor.at_derivation_index(4).script_pubkey();
+/// # let external_spk_0 = descriptor.at_derivation_index(0).unwrap().script_pubkey();
+/// # let external_spk_3 = descriptor.at_derivation_index(3).unwrap().script_pubkey();
+/// # let external_spk_4 = descriptor.at_derivation_index(4).unwrap().script_pubkey();
 ///
 /// // Creates a new script pubkey iterator starting at 0 from a descriptor.
 /// let mut spk_iter = SpkIterator::new(&descriptor);
@@ -43,48 +43,61 @@ impl<D> SpkIterator<D>
 where
     D: Borrow<Descriptor<DescriptorPublicKey>>,
 {
-    /// Creates a new script pubkey iterator starting at 0 from a descriptor.
+    /// Create a new script pubkey iterator from `descriptor`.
+    ///
+    /// This iterates from derivation index 0 and stops at index 0x7FFFFFFF (as specified in
+    /// BIP-32). Non-wildcard descriptors will only return one script pubkey at derivation index 0.
+    ///
+    /// Use [`new_with_range`](SpkIterator::new_with_range) to create an iterator with a specified
+    /// derivation index range.
     pub fn new(descriptor: D) -> Self {
-        let end = if descriptor.borrow().has_wildcard() {
-            BIP32_MAX_INDEX
-        } else {
-            0
-        };
-
-        SpkIterator::new_with_range(descriptor, 0..=end)
+        SpkIterator::new_with_range(descriptor, 0..=BIP32_MAX_INDEX)
     }
 
-    // Creates a new script pubkey iterator from a descriptor with a given range.
-    pub(crate) fn new_with_range<R>(descriptor: D, range: R) -> Self
+    /// Create a new script pubkey iterator from `descriptor` and a given `range`.
+    ///
+    /// Non-wildcard descriptors will only emit a single script pubkey (at derivation index 0).
+    /// Wildcard descriptors have an end-bound of 0x7FFFFFFF (inclusive).
+    ///
+    /// Refer to [`new`](SpkIterator::new) for more.
+    pub fn new_with_range<R>(descriptor: D, range: R) -> Self
     where
         R: RangeBounds<u32>,
     {
+        let start = match range.start_bound() {
+            Bound::Included(start) => *start,
+            Bound::Excluded(start) => *start + 1,
+            Bound::Unbounded => u32::MIN,
+        };
+
         let mut end = match range.end_bound() {
             Bound::Included(end) => *end + 1,
             Bound::Excluded(end) => *end,
             Bound::Unbounded => u32::MAX,
         };
+
         // Because `end` is exclusive, we want the maximum value to be BIP32_MAX_INDEX + 1.
         end = end.min(BIP32_MAX_INDEX + 1);
 
         Self {
-            next_index: match range.start_bound() {
-                Bound::Included(start) => *start,
-                Bound::Excluded(start) => *start + 1,
-                Bound::Unbounded => u32::MIN,
-            },
+            next_index: start,
             end,
             descriptor,
             secp: Secp256k1::verification_only(),
         }
     }
+
+    /// Get a reference to the internal descriptor.
+    pub fn descriptor(&self) -> &D {
+        &self.descriptor
+    }
 }
 
 impl<D> Iterator for SpkIterator<D>
 where
     D: Borrow<Descriptor<DescriptorPublicKey>>,
 {
-    type Item = (u32, Script);
+    type Item = (u32, ScriptBuf);
 
     fn next(&mut self) -> Option<Self::Item> {
         // For non-wildcard descriptors, we expect the first element to be Some((0, spk)), then None after.
@@ -93,11 +106,15 @@ where
             return None;
         }
 
+        // If the descriptor is non-wildcard, only index 0 will return an spk.
+        if !self.descriptor.borrow().has_wildcard() && self.next_index != 0 {
+            return None;
+        }
+
         let script = self
             .descriptor
             .borrow()
-            .at_derivation_index(self.next_index)
-            .derived_descriptor(&self.secp)
+            .derived_descriptor(&self.secp, self.next_index)
             .expect("the descriptor cannot need hardened derivation")
             .script_pubkey();
         let output = (self.next_index, script);
@@ -135,7 +152,7 @@ mod test {
         Descriptor<DescriptorPublicKey>,
         Descriptor<DescriptorPublicKey>,
     ) {
-        let mut txout_index = KeychainTxOutIndex::<TestKeychain>::default();
+        let mut txout_index = KeychainTxOutIndex::<TestKeychain>::new(0);
 
         let secp = Secp256k1::signing_only();
         let (external_descriptor,_) = Descriptor::<DescriptorPublicKey>::parse_descriptor(&secp, "tr([73c5da0a/86'/0'/0']xprv9xgqHN7yz9MwCkxsBPN5qetuNdQSUttZNKw1dcYTV4mkaAFiBVGQziHs3NRSWMkCzvgjEe3n9xV8oYywvM8at9yRqyaZVz6TYYhX98VjsUk/0/*)").unwrap();
@@ -149,31 +166,30 @@ mod test {
 
     #[test]
     #[allow(clippy::iter_nth_zero)]
+    #[rustfmt::skip]
     fn test_spkiterator_wildcard() {
         let (_, external_desc, _) = init_txout_index();
-        let external_spk_0 = external_desc.at_derivation_index(0).script_pubkey();
-        let external_spk_16 = external_desc.at_derivation_index(16).script_pubkey();
-        let external_spk_20 = external_desc.at_derivation_index(20).script_pubkey();
-        let external_spk_21 = external_desc.at_derivation_index(21).script_pubkey();
-        let external_spk_max = external_desc
-            .at_derivation_index(BIP32_MAX_INDEX)
-            .script_pubkey();
+        let external_spk_0 = external_desc.at_derivation_index(0).unwrap().script_pubkey();
+        let external_spk_16 = external_desc.at_derivation_index(16).unwrap().script_pubkey();
+        let external_spk_20 = external_desc.at_derivation_index(20).unwrap().script_pubkey();
+        let external_spk_21 = external_desc.at_derivation_index(21).unwrap().script_pubkey();
+        let external_spk_max = external_desc.at_derivation_index(BIP32_MAX_INDEX).unwrap().script_pubkey();
 
         let mut external_spk = SpkIterator::new(&external_desc);
         let max_index = BIP32_MAX_INDEX - 22;
 
-        assert_eq!(external_spk.next().unwrap(), (0, external_spk_0));
-        assert_eq!(external_spk.nth(15).unwrap(), (16, external_spk_16));
-        assert_eq!(external_spk.nth(3).unwrap(), (20, external_spk_20.clone()));
-        assert_eq!(external_spk.next().unwrap(), (21, external_spk_21));
+        assert_eq!(external_spk.next(), Some((0, external_spk_0)));
+        assert_eq!(external_spk.nth(15), Some((16, external_spk_16)));
+        assert_eq!(external_spk.nth(3), Some((20, external_spk_20.clone())));
+        assert_eq!(external_spk.next(), Some((21, external_spk_21)));
         assert_eq!(
-            external_spk.nth(max_index as usize).unwrap(),
-            (BIP32_MAX_INDEX, external_spk_max)
+            external_spk.nth(max_index as usize),
+            Some((BIP32_MAX_INDEX, external_spk_max))
         );
         assert_eq!(external_spk.nth(0), None);
 
         let mut external_spk = SpkIterator::new_with_range(&external_desc, 0..21);
-        assert_eq!(external_spk.nth(20).unwrap(), (20, external_spk_20));
+        assert_eq!(external_spk.nth(20), Some((20, external_spk_20)));
         assert_eq!(external_spk.next(), None);
 
         let mut external_spk = SpkIterator::new_with_range(&external_desc, 0..21);
@@ -187,17 +203,60 @@ mod test {
         let (no_wildcard_descriptor, _) = Descriptor::<DescriptorPublicKey>::parse_descriptor(&secp, "wpkh([73c5da0a/86'/0'/0']xprv9xgqHN7yz9MwCkxsBPN5qetuNdQSUttZNKw1dcYTV4mkaAFiBVGQziHs3NRSWMkCzvgjEe3n9xV8oYywvM8at9yRqyaZVz6TYYhX98VjsUk/1/0)").unwrap();
         let external_spk_0 = no_wildcard_descriptor
             .at_derivation_index(0)
+            .unwrap()
             .script_pubkey();
 
         let mut external_spk = SpkIterator::new(&no_wildcard_descriptor);
 
-        assert_eq!(external_spk.next().unwrap(), (0, external_spk_0.clone()));
+        assert_eq!(external_spk.next(), Some((0, external_spk_0.clone())));
         assert_eq!(external_spk.next(), None);
 
         let mut external_spk = SpkIterator::new(&no_wildcard_descriptor);
 
-        assert_eq!(external_spk.nth(0).unwrap(), (0, external_spk_0));
+        assert_eq!(external_spk.nth(0), Some((0, external_spk_0.clone())));
         assert_eq!(external_spk.nth(0), None);
+
+        let mut external_spk = SpkIterator::new_with_range(&no_wildcard_descriptor, 0..0);
+
+        assert_eq!(external_spk.next(), None);
+
+        let mut external_spk = SpkIterator::new_with_range(&no_wildcard_descriptor, 0..1);
+
+        assert_eq!(external_spk.nth(0), Some((0, external_spk_0.clone())));
+        assert_eq!(external_spk.next(), None);
+
+        // We test that using new_with_range with range_len > 1 gives back an iterator with
+        // range_len = 1
+        let mut external_spk = SpkIterator::new_with_range(&no_wildcard_descriptor, 0..10);
+
+        assert_eq!(external_spk.nth(0), Some((0, external_spk_0)));
+        assert_eq!(external_spk.nth(0), None);
+
+        // non index-0 should NOT return an spk
+        assert_eq!(
+            SpkIterator::new_with_range(&no_wildcard_descriptor, 1..1).next(),
+            None
+        );
+        assert_eq!(
+            SpkIterator::new_with_range(&no_wildcard_descriptor, 1..=1).next(),
+            None
+        );
+        assert_eq!(
+            SpkIterator::new_with_range(&no_wildcard_descriptor, 1..2).next(),
+            None
+        );
+        assert_eq!(
+            SpkIterator::new_with_range(&no_wildcard_descriptor, 1..=2).next(),
+            None
+        );
+        assert_eq!(
+            SpkIterator::new_with_range(&no_wildcard_descriptor, 10..11).next(),
+            None
+        );
+        assert_eq!(
+            SpkIterator::new_with_range(&no_wildcard_descriptor, 10..=10).next(),
+            None
+        );
     }
 
     // The following dummy traits were created to test if SpkIterator is working properly.

crates/chain/src/spk_txout_index.rs

@@ -3,15 +3,15 @@ use core::ops::RangeBounds;
 use crate::{
     collections::{hash_map::Entry, BTreeMap, BTreeSet, HashMap},
     indexed_tx_graph::Indexer,
-    ForEachTxOut,
 };
-use bitcoin::{self, OutPoint, Script, Transaction, TxOut, Txid};
+use bitcoin::{self, OutPoint, Script, ScriptBuf, Transaction, TxOut, Txid};
 
 /// An index storing [`TxOut`]s that have a script pubkey that matches those in a list.
 ///
 /// The basic idea is that you insert script pubkeys you care about into the index with
-/// [`insert_spk`] and then when you call [`scan`], the index will look at any txouts you pass in and
-/// store and index any txouts matching one of its script pubkeys.
+/// [`insert_spk`] and then when you call [`Indexer::index_tx`] or [`Indexer::index_txout`], the
+/// index will look at any txouts you pass in and store and index any txouts matching one of its
+/// script pubkeys.
 ///
 /// Each script pubkey is associated with an application-defined index script index `I`, which must be
 /// [`Ord`]. Usually, this is used to associate the derivation index of the script pubkey or even a
@@ -25,14 +25,13 @@ use bitcoin::{self, OutPoint, Script, Transaction, TxOut, Txid};
 /// [`TxOut`]: bitcoin::TxOut
 /// [`insert_spk`]: Self::insert_spk
 /// [`Ord`]: core::cmp::Ord
-/// [`scan`]: Self::scan
 /// [`TxGraph`]: crate::tx_graph::TxGraph
 #[derive(Clone, Debug)]
 pub struct SpkTxOutIndex<I> {
     /// script pubkeys ordered by index
-    spks: BTreeMap<I, Script>,
+    spks: BTreeMap<I, ScriptBuf>,
     /// A reverse lookup from spk to spk index
-    spk_indices: HashMap<Script, I>,
+    spk_indices: HashMap<ScriptBuf, I>,
     /// The set of unused indexes.
     unused: BTreeSet<I>,
     /// Lookup index and txout by outpoint.
@@ -54,19 +53,21 @@ impl<I> Default for SpkTxOutIndex<I> {
 }
 
 impl<I: Clone + Ord> Indexer for SpkTxOutIndex<I> {
-    type Additions = ();
+    type ChangeSet = ();
 
-    fn index_txout(&mut self, outpoint: OutPoint, txout: &TxOut) -> Self::Additions {
+    fn index_txout(&mut self, outpoint: OutPoint, txout: &TxOut) -> Self::ChangeSet {
         self.scan_txout(outpoint, txout);
         Default::default()
     }
 
-    fn index_tx(&mut self, tx: &Transaction) -> Self::Additions {
+    fn index_tx(&mut self, tx: &Transaction) -> Self::ChangeSet {
         self.scan(tx);
         Default::default()
     }
 
-    fn apply_additions(&mut self, _additions: Self::Additions) {
+    fn initial_changeset(&self) -> Self::ChangeSet {}
+
+    fn apply_changeset(&mut self, _changeset: Self::ChangeSet) {
         // This applies nothing.
     }
 
@@ -75,41 +76,23 @@ impl<I: Clone + Ord> Indexer for SpkTxOutIndex<I> {
     }
 }
 
-/// This macro is used instead of a member function of `SpkTxOutIndex`, which would result in a
-/// compiler error[E0521]: "borrowed data escapes out of closure" when we attempt to take a
-/// reference out of the `ForEachTxOut` closure during scanning.
-macro_rules! scan_txout {
-    ($self:ident, $op:expr, $txout:expr) => {{
-        let spk_i = $self.spk_indices.get(&$txout.script_pubkey);
-        if let Some(spk_i) = spk_i {
-            $self.txouts.insert($op, (spk_i.clone(), $txout.clone()));
-            $self.spk_txouts.insert((spk_i.clone(), $op));
-            $self.unused.remove(&spk_i);
-        }
-        spk_i
-    }};
-}
-
 impl<I: Clone + Ord> SpkTxOutIndex<I> {
-    /// Scans an object containing many txouts.
+    /// Scans a transaction's outputs for matching script pubkeys.
     ///
     /// Typically, this is used in two situations:
     ///
     /// 1. After loading transaction data from the disk, you may scan over all the txouts to restore all
     /// your txouts.
     /// 2. When getting new data from the chain, you usually scan it before incorporating it into your chain state.
-    ///
-    /// See [`ForEachTxout`] for the types that support this.
-    ///
-    /// [`ForEachTxout`]: crate::ForEachTxOut
-    pub fn scan(&mut self, txouts: &impl ForEachTxOut) -> BTreeSet<I> {
+    pub fn scan(&mut self, tx: &Transaction) -> BTreeSet<I> {
         let mut scanned_indices = BTreeSet::new();
-
-        txouts.for_each_txout(|(op, txout)| {
-            if let Some(spk_i) = scan_txout!(self, op, txout) {
+        let txid = tx.txid();
+        for (i, txout) in tx.output.iter().enumerate() {
+            let op = OutPoint::new(txid, i as u32);
+            if let Some(spk_i) = self.scan_txout(op, txout) {
                 scanned_indices.insert(spk_i.clone());
             }
-        });
+        }
 
         scanned_indices
     }
@@ -117,7 +100,13 @@ impl<I: Clone + Ord> SpkTxOutIndex<I> {
     /// Scan a single `TxOut` for a matching script pubkey and returns the index that matches the
     /// script pubkey (if any).
     pub fn scan_txout(&mut self, op: OutPoint, txout: &TxOut) -> Option<&I> {
-        scan_txout!(self, op, txout)
+        let spk_i = self.spk_indices.get(&txout.script_pubkey);
+        if let Some(spk_i) = spk_i {
+            self.txouts.insert(op, (spk_i.clone(), txout.clone()));
+            self.spk_txouts.insert((spk_i.clone(), op));
+            self.unused.remove(spk_i);
+        }
+        spk_i
     }
 
     /// Get a reference to the set of indexed outpoints.
@@ -152,11 +141,11 @@ impl<I: Clone + Ord> SpkTxOutIndex<I> {
         use bitcoin::hashes::Hash;
         use core::ops::Bound::*;
         let min_op = OutPoint {
-            txid: Txid::from_inner([0x00; 32]),
+            txid: Txid::all_zeros(),
             vout: u32::MIN,
         };
         let max_op = OutPoint {
-            txid: Txid::from_inner([0xff; 32]),
+            txid: Txid::from_byte_array([0xff; Txid::LEN]),
             vout: u32::MAX,
         };
 
@@ -179,27 +168,25 @@ impl<I: Clone + Ord> SpkTxOutIndex<I> {
     ///
     /// Returns `None` if the `TxOut` hasn't been scanned or if nothing matching was found there.
     pub fn txout(&self, outpoint: OutPoint) -> Option<(&I, &TxOut)> {
-        self.txouts
-            .get(&outpoint)
-            .map(|(spk_i, txout)| (spk_i, txout))
+        self.txouts.get(&outpoint).map(|v| (&v.0, &v.1))
     }
 
     /// Returns the script that has been inserted at the `index`.
     ///
     /// If that index hasn't been inserted yet, it will return `None`.
     pub fn spk_at_index(&self, index: &I) -> Option<&Script> {
-        self.spks.get(index)
+        self.spks.get(index).map(|s| s.as_script())
     }
 
     /// The script pubkeys that are being tracked by the index.
-    pub fn all_spks(&self) -> &BTreeMap<I, Script> {
+    pub fn all_spks(&self) -> &BTreeMap<I, ScriptBuf> {
         &self.spks
     }
 
     /// Adds a script pubkey to scan for. Returns `false` and does nothing if spk already exists in the map
     ///
     /// the index will look for outputs spending to this spk whenever it scans new data.
-    pub fn insert_spk(&mut self, index: I, spk: Script) -> bool {
+    pub fn insert_spk(&mut self, index: I, spk: ScriptBuf) -> bool {
         match self.spk_indices.entry(spk.clone()) {
             Entry::Vacant(value) => {
                 value.insert(index.clone());
@@ -228,7 +215,7 @@ impl<I: Clone + Ord> SpkTxOutIndex<I> {
     /// let unused_change_spks =
     ///     txout_index.unused_spks((change_index, u32::MIN)..(change_index, u32::MAX));
     /// ```
-    pub fn unused_spks<R>(&self, range: R) -> impl DoubleEndedIterator<Item = (&I, &Script)>
+    pub fn unused_spks<R>(&self, range: R) -> impl DoubleEndedIterator<Item = (&I, &Script)> + Clone
     where
         R: RangeBounds<I>,
     {
@@ -286,8 +273,8 @@ impl<I: Clone + Ord> SpkTxOutIndex<I> {
     /// Computes total input value going from script pubkeys in the index (sent) and the total output
     /// value going to script pubkeys in the index (received) in `tx`. For the `sent` to be computed
     /// correctly, the output being spent must have already been scanned by the index. Calculating
-    /// received just uses the transaction outputs directly, so it will be correct even if it has not
-    /// been scanned.
+    /// received just uses the [`Transaction`] outputs directly, so it will be correct even if it has
+    /// not been scanned.
     pub fn sent_and_received(&self, tx: &Transaction) -> (u64, u64) {
         let mut sent = 0;
         let mut received = 0;

crates/chain/src/tx_data_traits.rs

@@ -2,46 +2,91 @@ use crate::collections::BTreeMap;
 use crate::collections::BTreeSet;
 use crate::BlockId;
 use alloc::vec::Vec;
-use bitcoin::{Block, OutPoint, Transaction, TxOut};
-
-/// Trait to do something with every txout contained in a structure.
-///
-/// We would prefer to just work with things that can give us an `Iterator<Item=(OutPoint, &TxOut)>`
-/// here, but rust's type system makes it extremely hard to do this (without trait objects).
-pub trait ForEachTxOut {
-    /// The provided closure `f` will be called with each `outpoint/txout` pair.
-    fn for_each_txout(&self, f: impl FnMut((OutPoint, &TxOut)));
-}
-
-impl ForEachTxOut for Block {
-    fn for_each_txout(&self, mut f: impl FnMut((OutPoint, &TxOut))) {
-        for tx in self.txdata.iter() {
-            tx.for_each_txout(&mut f)
-        }
-    }
-}
-
-impl ForEachTxOut for Transaction {
-    fn for_each_txout(&self, mut f: impl FnMut((OutPoint, &TxOut))) {
-        let txid = self.txid();
-        for (i, txout) in self.output.iter().enumerate() {
-            f((
-                OutPoint {
-                    txid,
-                    vout: i as u32,
-                },
-                txout,
-            ))
-        }
-    }
-}
 
 /// Trait that "anchors" blockchain data to a specific block of height and hash.
 ///
-/// I.e. If transaction A is anchored in block B, then if block B is in the best chain, we can
+/// If transaction A is anchored in block B, and block B is in the best chain, we can
 /// assume that transaction A is also confirmed in the best chain. This does not necessarily mean
 /// that transaction A is confirmed in block B. It could also mean transaction A is confirmed in a
 /// parent block of B.
+///
+/// Every [`Anchor`] implementation must contain a [`BlockId`] parameter, and must implement
+/// [`Ord`]. When implementing [`Ord`], the anchors' [`BlockId`]s should take precedence
+/// over other elements inside the [`Anchor`]s for comparison purposes, i.e., you should first
+/// compare the anchors' [`BlockId`]s and then care about the rest.
+///
+/// The example shows different types of anchors:
+/// ```
+/// # use bdk_chain::local_chain::LocalChain;
+/// # use bdk_chain::tx_graph::TxGraph;
+/// # use bdk_chain::BlockId;
+/// # use bdk_chain::ConfirmationHeightAnchor;
+/// # use bdk_chain::ConfirmationTimeHeightAnchor;
+/// # use bdk_chain::example_utils::*;
+/// # use bitcoin::hashes::Hash;
+/// // Initialize the local chain with two blocks.
+/// let chain = LocalChain::from_blocks(
+///     [
+///         (1, Hash::hash("first".as_bytes())),
+///         (2, Hash::hash("second".as_bytes())),
+///     ]
+///     .into_iter()
+///     .collect(),
+/// );
+///
+/// // Transaction to be inserted into `TxGraph`s with different anchor types.
+/// let tx = tx_from_hex(RAW_TX_1);
+///
+/// // Insert `tx` into a `TxGraph` that uses `BlockId` as the anchor type.
+/// // When a transaction is anchored with `BlockId`, the anchor block and the confirmation block of
+/// // the transaction is the same block.
+/// let mut graph_a = TxGraph::<BlockId>::default();
+/// let _ = graph_a.insert_tx(tx.clone());
+/// graph_a.insert_anchor(
+///     tx.txid(),
+///     BlockId {
+///         height: 1,
+///         hash: Hash::hash("first".as_bytes()),
+///     },
+/// );
+///
+/// // Insert `tx` into a `TxGraph` that uses `ConfirmationHeightAnchor` as the anchor type.
+/// // This anchor records the anchor block and the confirmation height of the transaction.
+/// // When a transaction is anchored with `ConfirmationHeightAnchor`, the anchor block and
+/// // confirmation block can be different. However, the confirmation block cannot be higher than
+/// // the anchor block and both blocks must be in the same chain for the anchor to be valid.
+/// let mut graph_b = TxGraph::<ConfirmationHeightAnchor>::default();
+/// let _ = graph_b.insert_tx(tx.clone());
+/// graph_b.insert_anchor(
+///     tx.txid(),
+///     ConfirmationHeightAnchor {
+///         anchor_block: BlockId {
+///             height: 2,
+///             hash: Hash::hash("second".as_bytes()),
+///         },
+///         confirmation_height: 1,
+///     },
+/// );
+///
+/// // Insert `tx` into a `TxGraph` that uses `ConfirmationTimeHeightAnchor` as the anchor type.
+/// // This anchor records the anchor block, the confirmation height and time of the transaction.
+/// // When a transaction is anchored with `ConfirmationTimeHeightAnchor`, the anchor block and
+/// // confirmation block can be different. However, the confirmation block cannot be higher than
+/// // the anchor block and both blocks must be in the same chain for the anchor to be valid.
+/// let mut graph_c = TxGraph::<ConfirmationTimeHeightAnchor>::default();
+/// let _ = graph_c.insert_tx(tx.clone());
+/// graph_c.insert_anchor(
+///     tx.txid(),
+///     ConfirmationTimeHeightAnchor {
+///         anchor_block: BlockId {
+///             height: 2,
+///             hash: Hash::hash("third".as_bytes()),
+///         },
+///         confirmation_height: 1,
+///         confirmation_time: 123,
+///     },
+/// );
+/// ```
 pub trait Anchor: core::fmt::Debug + Clone + Eq + PartialOrd + Ord + core::hash::Hash {
     /// Returns the [`BlockId`] that the associated blockchain data is "anchored" in.
     fn anchor_block(&self) -> BlockId;
@@ -55,12 +100,19 @@ pub trait Anchor: core::fmt::Debug + Clone + Eq + PartialOrd + Ord + core::hash:
     }
 }
 
-impl<A: Anchor> Anchor for &'static A {
+impl<'a, A: Anchor> Anchor for &'a A {
     fn anchor_block(&self) -> BlockId {
         <A as Anchor>::anchor_block(self)
     }
 }
 
+/// An [`Anchor`] that can be constructed from a given block, block height and transaction position
+/// within the block.
+pub trait AnchorFromBlockPosition: Anchor {
+    /// Construct the anchor from a given `block`, block height and `tx_pos` within the block.
+    fn from_block_position(block: &bitcoin::Block, block_id: BlockId, tx_pos: usize) -> Self;
+}
+
 /// Trait that makes an object appendable.
 pub trait Append {
     /// Append another object of the same type onto `self`.
@@ -70,17 +122,11 @@ pub trait Append {
     fn is_empty(&self) -> bool;
 }
 
-impl Append for () {
-    fn append(&mut self, _other: Self) {}
-
-    fn is_empty(&self) -> bool {
-        true
-    }
-}
-
 impl<K: Ord, V> Append for BTreeMap<K, V> {
-    fn append(&mut self, mut other: Self) {
-        BTreeMap::append(self, &mut other)
+    fn append(&mut self, other: Self) {
+        // We use `extend` instead of `BTreeMap::append` due to performance issues with `append`.
+        // Refer to https://github.com/rust-lang/rust/issues/34666#issuecomment-675658420
+        BTreeMap::extend(self, other)
     }
 
     fn is_empty(&self) -> bool {
@@ -89,8 +135,10 @@ impl<K: Ord, V> Append for BTreeMap<K, V> {
 }
 
 impl<T: Ord> Append for BTreeSet<T> {
-    fn append(&mut self, mut other: Self) {
-        BTreeSet::append(self, &mut other)
+    fn append(&mut self, other: Self) {
+        // We use `extend` instead of `BTreeMap::append` due to performance issues with `append`.
+        // Refer to https://github.com/rust-lang/rust/issues/34666#issuecomment-675658420
+        BTreeSet::extend(self, other)
     }
 
     fn is_empty(&self) -> bool {
@@ -108,13 +156,30 @@ impl<T> Append for Vec<T> {
     }
 }
 
-impl<A: Append, B: Append> Append for (A, B) {
-    fn append(&mut self, other: Self) {
-        Append::append(&mut self.0, other.0);
-        Append::append(&mut self.1, other.1);
-    }
+macro_rules! impl_append_for_tuple {
+    ($($a:ident $b:tt)*) => {
+        impl<$($a),*> Append for ($($a,)*) where $($a: Append),* {
 
-    fn is_empty(&self) -> bool {
-        Append::is_empty(&self.0) && Append::is_empty(&self.1)
+            fn append(&mut self, _other: Self) {
+                $(Append::append(&mut self.$b, _other.$b) );*
+            }
+
+            fn is_empty(&self) -> bool {
+                $(Append::is_empty(&self.$b) && )* true
+            }
+        }
     }
 }
+
+impl_append_for_tuple!();
+impl_append_for_tuple!(T0 0);
+impl_append_for_tuple!(T0 0 T1 1);
+impl_append_for_tuple!(T0 0 T1 1 T2 2);
+impl_append_for_tuple!(T0 0 T1 1 T2 2 T3 3);
+impl_append_for_tuple!(T0 0 T1 1 T2 2 T3 3 T4 4);
+impl_append_for_tuple!(T0 0 T1 1 T2 2 T3 3 T4 4 T5 5);
+impl_append_for_tuple!(T0 0 T1 1 T2 2 T3 3 T4 4 T5 5 T6 6);
+impl_append_for_tuple!(T0 0 T1 1 T2 2 T3 3 T4 4 T5 5 T6 6 T7 7);
+impl_append_for_tuple!(T0 0 T1 1 T2 2 T3 3 T4 4 T5 5 T6 6 T7 7 T8 8);
+impl_append_for_tuple!(T0 0 T1 1 T2 2 T3 3 T4 4 T5 5 T6 6 T7 7 T8 8 T9 9);
+impl_append_for_tuple!(T0 0 T1 1 T2 2 T3 3 T4 4 T5 5 T6 6 T7 7 T8 8 T9 9 T10 10);

crates/chain/tests/common/mod.rs

@@ -1,3 +1,17 @@
+mod tx_template;
+#[allow(unused_imports)]
+pub use tx_template::*;
+
+#[allow(unused_macros)]
+macro_rules! block_id {
+    ($height:expr, $hash:literal) => {{
+        bdk_chain::BlockId {
+            height: $height,
+            hash: bitcoin::hashes::Hash::hash($hash.as_bytes()),
+        }
+    }};
+}
+
 #[allow(unused_macros)]
 macro_rules! h {
     ($index:literal) => {{
@@ -9,25 +23,21 @@ macro_rules! h {
 macro_rules! local_chain {
     [ $(($height:expr, $block_hash:expr)), * ] => {{
         #[allow(unused_mut)]
-        bdk_chain::local_chain::LocalChain::from_blocks([$(($height, $block_hash).into()),*])
+        bdk_chain::local_chain::LocalChain::from_blocks([$(($height, $block_hash).into()),*].into_iter().collect())
+            .expect("chain must have genesis block")
     }};
 }
 
 #[allow(unused_macros)]
-macro_rules! chain {
-    ($([$($tt:tt)*]),*) => { chain!( checkpoints: [$([$($tt)*]),*] ) };
-    (checkpoints: $($tail:tt)*) => { chain!( index: TxHeight, checkpoints: $($tail)*) };
-    (index: $ind:ty, checkpoints: [ $([$height:expr, $block_hash:expr]),* ] $(,txids: [$(($txid:expr, $tx_height:expr)),*])?) => {{
+macro_rules! chain_update {
+    [ $(($height:expr, $hash:expr)), * ] => {{
         #[allow(unused_mut)]
-        let mut chain = bdk_chain::sparse_chain::SparseChain::<$ind>::from_checkpoints([$(($height, $block_hash).into()),*]);
-
-        $(
-            $(
-                let _ = chain.insert_tx($txid, $tx_height).expect("should succeed");
-            )*
-        )?
-
-        chain
+        bdk_chain::local_chain::Update {
+            tip: bdk_chain::local_chain::LocalChain::from_blocks([$(($height, $hash).into()),*].into_iter().collect())
+                .expect("chain must have genesis block")
+                .tip(),
+            introduce_older_blocks: true,
+        }
     }};
 }
 
@@ -61,7 +71,7 @@ macro_rules! changeset {
 pub fn new_tx(lt: u32) -> bitcoin::Transaction {
     bitcoin::Transaction {
         version: 0x00,
-        lock_time: bitcoin::PackedLockTime(lt),
+        lock_time: bitcoin::absolute::LockTime::from_consensus(lt),
         input: vec![],
         output: vec![],
     }

crates/chain/tests/common/tx_template.rs

@@ -0,0 +1,136 @@
+use rand::distributions::{Alphanumeric, DistString};
+use std::collections::HashMap;
+
+use bdk_chain::{tx_graph::TxGraph, Anchor, SpkTxOutIndex};
+use bitcoin::{
+    locktime::absolute::LockTime, secp256k1::Secp256k1, OutPoint, ScriptBuf, Sequence, Transaction,
+    TxIn, TxOut, Txid, Witness,
+};
+use miniscript::Descriptor;
+
+/// Template for creating a transaction in `TxGraph`.
+///
+/// The incentive for transaction templates is to create a transaction history in a simple manner to
+/// avoid having to explicitly hash previous transactions to form previous outpoints of later
+/// transactions.
+#[derive(Clone, Copy, Default)]
+pub struct TxTemplate<'a, A> {
+    /// Uniquely identifies the transaction, before it can have a txid.
+    pub tx_name: &'a str,
+    pub inputs: &'a [TxInTemplate<'a>],
+    pub outputs: &'a [TxOutTemplate],
+    pub anchors: &'a [A],
+    pub last_seen: Option<u64>,
+}
+
+#[allow(dead_code)]
+pub enum TxInTemplate<'a> {
+    /// This will give a random txid and vout.
+    Bogus,
+
+    /// This is used for coinbase transactions because they do not have previous outputs.
+    Coinbase,
+
+    /// Contains the `tx_name` and `vout` that we are spending. The rule is that we must only spend
+    /// from tx of a previous `TxTemplate`.
+    PrevTx(&'a str, usize),
+}
+
+pub struct TxOutTemplate {
+    pub value: u64,
+    pub spk_index: Option<u32>, // some = get spk from SpkTxOutIndex, none = random spk
+}
+
+#[allow(unused)]
+impl TxOutTemplate {
+    pub fn new(value: u64, spk_index: Option<u32>) -> Self {
+        TxOutTemplate { value, spk_index }
+    }
+}
+
+#[allow(dead_code)]
+pub fn init_graph<'a, A: Anchor + Clone + 'a>(
+    tx_templates: impl IntoIterator<Item = &'a TxTemplate<'a, A>>,
+) -> (TxGraph<A>, SpkTxOutIndex<u32>, HashMap<&'a str, Txid>) {
+    let (descriptor, _) = Descriptor::parse_descriptor(&Secp256k1::signing_only(), "tr(tprv8ZgxMBicQKsPd3krDUsBAmtnRsK3rb8u5yi1zhQgMhF1tR8MW7xfE4rnrbbsrbPR52e7rKapu6ztw1jXveJSCGHEriUGZV7mCe88duLp5pj/86'/1'/0'/0/*)").unwrap();
+    let mut graph = TxGraph::<A>::default();
+    let mut spk_index = SpkTxOutIndex::default();
+    (0..10).for_each(|index| {
+        spk_index.insert_spk(
+            index,
+            descriptor
+                .at_derivation_index(index)
+                .unwrap()
+                .script_pubkey(),
+        );
+    });
+    let mut tx_ids = HashMap::<&'a str, Txid>::new();
+
+    for (bogus_txin_vout, tx_tmp) in tx_templates.into_iter().enumerate() {
+        let tx = Transaction {
+            version: 0,
+            lock_time: LockTime::ZERO,
+            input: tx_tmp
+                .inputs
+                .iter()
+                .map(|input| match input {
+                    TxInTemplate::Bogus => TxIn {
+                        previous_output: OutPoint::new(
+                            bitcoin::hashes::Hash::hash(
+                                Alphanumeric
+                                    .sample_string(&mut rand::thread_rng(), 20)
+                                    .as_bytes(),
+                            ),
+                            bogus_txin_vout as u32,
+                        ),
+                        script_sig: ScriptBuf::new(),
+                        sequence: Sequence::default(),
+                        witness: Witness::new(),
+                    },
+                    TxInTemplate::Coinbase => TxIn {
+                        previous_output: OutPoint::null(),
+                        script_sig: ScriptBuf::new(),
+                        sequence: Sequence::MAX,
+                        witness: Witness::new(),
+                    },
+                    TxInTemplate::PrevTx(prev_name, prev_vout) => {
+                        let prev_txid = tx_ids.get(prev_name).expect(
+                            "txin template must spend from tx of template that comes before",
+                        );
+                        TxIn {
+                            previous_output: OutPoint::new(*prev_txid, *prev_vout as _),
+                            script_sig: ScriptBuf::new(),
+                            sequence: Sequence::default(),
+                            witness: Witness::new(),
+                        }
+                    }
+                })
+                .collect(),
+            output: tx_tmp
+                .outputs
+                .iter()
+                .map(|output| match &output.spk_index {
+                    None => TxOut {
+                        value: output.value,
+                        script_pubkey: ScriptBuf::new(),
+                    },
+                    Some(index) => TxOut {
+                        value: output.value,
+                        script_pubkey: spk_index.spk_at_index(index).unwrap().to_owned(),
+                    },
+                })
+                .collect(),
+        };
+
+        tx_ids.insert(tx_tmp.tx_name, tx.txid());
+        spk_index.scan(&tx);
+        let _ = graph.insert_tx(tx.clone());
+        for anchor in tx_tmp.anchors.iter() {
+            let _ = graph.insert_anchor(tx.txid(), anchor.clone());
+        }
+        if let Some(seen_at) = tx_tmp.last_seen {
+            let _ = graph.insert_seen_at(tx.txid(), seen_at);
+        }
+    }
+    (graph, spk_index, tx_ids)
+}

crates/chain/tests/test_indexed_tx_graph.rs

@@ -1,16 +1,15 @@
 #[macro_use]
 mod common;
 
-use std::collections::{BTreeMap, BTreeSet};
+use std::collections::BTreeSet;
 
 use bdk_chain::{
-    indexed_tx_graph::{IndexedAdditions, IndexedTxGraph},
-    keychain::{Balance, DerivationAdditions, KeychainTxOutIndex},
+    indexed_tx_graph::{self, IndexedTxGraph},
+    keychain::{self, Balance, KeychainTxOutIndex},
     local_chain::LocalChain,
-    tx_graph::Additions,
-    BlockId, ChainPosition, ConfirmationHeightAnchor,
+    tx_graph, BlockId, ChainPosition, ConfirmationHeightAnchor,
 };
-use bitcoin::{secp256k1::Secp256k1, BlockHash, OutPoint, Script, Transaction, TxIn, TxOut};
+use bitcoin::{secp256k1::Secp256k1, OutPoint, Script, ScriptBuf, Transaction, TxIn, TxOut};
 use miniscript::Descriptor;
 
 /// Ensure [`IndexedTxGraph::insert_relevant_txs`] can successfully index transactions NOT presented
@@ -25,12 +24,13 @@ fn insert_relevant_txs() {
     const DESCRIPTOR: &str = "tr([73c5da0a/86'/0'/0']xprv9xgqHN7yz9MwCkxsBPN5qetuNdQSUttZNKw1dcYTV4mkaAFiBVGQziHs3NRSWMkCzvgjEe3n9xV8oYywvM8at9yRqyaZVz6TYYhX98VjsUk/0/*)";
     let (descriptor, _) = Descriptor::parse_descriptor(&Secp256k1::signing_only(), DESCRIPTOR)
         .expect("must be valid");
-    let spk_0 = descriptor.at_derivation_index(0).script_pubkey();
-    let spk_1 = descriptor.at_derivation_index(9).script_pubkey();
+    let spk_0 = descriptor.at_derivation_index(0).unwrap().script_pubkey();
+    let spk_1 = descriptor.at_derivation_index(9).unwrap().script_pubkey();
 
-    let mut graph = IndexedTxGraph::<ConfirmationHeightAnchor, KeychainTxOutIndex<()>>::default();
+    let mut graph = IndexedTxGraph::<ConfirmationHeightAnchor, KeychainTxOutIndex<()>>::new(
+        KeychainTxOutIndex::new(10),
+    );
     graph.index.add_keychain((), descriptor);
-    graph.index.set_lookahead(&(), 10);
 
     let tx_a = Transaction {
         output: vec![
@@ -64,16 +64,20 @@ fn insert_relevant_txs() {
 
     let txs = [tx_c, tx_b, tx_a];
 
+    let changeset = indexed_tx_graph::ChangeSet {
+        graph: tx_graph::ChangeSet {
+            txs: txs.clone().into(),
+            ..Default::default()
+        },
+        indexer: keychain::ChangeSet([((), 9_u32)].into()),
+    };
+
     assert_eq!(
-        graph.insert_relevant_txs(txs.iter().map(|tx| (tx, None)), None),
-        IndexedAdditions {
-            graph_additions: Additions {
-                txs: txs.into(),
-                ..Default::default()
-            },
-            index_additions: DerivationAdditions([((), 9_u32)].into()),
-        }
-    )
+        graph.batch_insert_relevant(txs.iter().map(|tx| (tx, None))),
+        changeset,
+    );
+
+    assert_eq!(graph.initial_changeset(), changeset,);
 }
 
 #[test]
@@ -107,41 +111,38 @@ fn insert_relevant_txs() {
 
 fn test_list_owned_txouts() {
     // Create Local chains
-
-    let local_chain = (0..150)
-        .map(|i| (i as u32, h!("random")))
-        .collect::<BTreeMap<u32, BlockHash>>();
-    let local_chain = LocalChain::from(local_chain);
+    let local_chain = LocalChain::from_blocks((0..150).map(|i| (i as u32, h!("random"))).collect())
+        .expect("must have genesis hash");
 
     // Initiate IndexedTxGraph
 
     let (desc_1, _) = Descriptor::parse_descriptor(&Secp256k1::signing_only(), "tr(tprv8ZgxMBicQKsPd3krDUsBAmtnRsK3rb8u5yi1zhQgMhF1tR8MW7xfE4rnrbbsrbPR52e7rKapu6ztw1jXveJSCGHEriUGZV7mCe88duLp5pj/86'/1'/0'/0/*)").unwrap();
     let (desc_2, _) = Descriptor::parse_descriptor(&Secp256k1::signing_only(), "tr(tprv8ZgxMBicQKsPd3krDUsBAmtnRsK3rb8u5yi1zhQgMhF1tR8MW7xfE4rnrbbsrbPR52e7rKapu6ztw1jXveJSCGHEriUGZV7mCe88duLp5pj/86'/1'/0'/1/*)").unwrap();
 
-    let mut graph =
-        IndexedTxGraph::<ConfirmationHeightAnchor, KeychainTxOutIndex<String>>::default();
+    let mut graph = IndexedTxGraph::<ConfirmationHeightAnchor, KeychainTxOutIndex<String>>::new(
+        KeychainTxOutIndex::new(10),
+    );
 
     graph.index.add_keychain("keychain_1".into(), desc_1);
     graph.index.add_keychain("keychain_2".into(), desc_2);
-    graph.index.set_lookahead_for_all(10);
 
     // Get trusted and untrusted addresses
 
-    let mut trusted_spks = Vec::new();
-    let mut untrusted_spks = Vec::new();
+    let mut trusted_spks: Vec<ScriptBuf> = Vec::new();
+    let mut untrusted_spks: Vec<ScriptBuf> = Vec::new();
 
     {
         // we need to scope here to take immutanble reference of the graph
         for _ in 0..10 {
             let ((_, script), _) = graph.index.reveal_next_spk(&"keychain_1".to_string());
             // TODO Assert indexes
-            trusted_spks.push(script.clone());
+            trusted_spks.push(script.to_owned());
         }
     }
     {
         for _ in 0..10 {
             let ((_, script), _) = graph.index.reveal_next_spk(&"keychain_2".to_string());
-            untrusted_spks.push(script.clone());
+            untrusted_spks.push(script.to_owned());
         }
     }
 
@@ -155,7 +156,7 @@ fn test_list_owned_txouts() {
         }],
         output: vec![TxOut {
             value: 70000,
-            script_pubkey: trusted_spks[0].clone(),
+            script_pubkey: trusted_spks[0].to_owned(),
         }],
         ..common::new_tx(0)
     };
@@ -164,7 +165,7 @@ fn test_list_owned_txouts() {
     let tx2 = Transaction {
         output: vec![TxOut {
             value: 30000,
-            script_pubkey: untrusted_spks[0].clone(),
+            script_pubkey: untrusted_spks[0].to_owned(),
         }],
         ..common::new_tx(0)
     };
@@ -177,7 +178,7 @@ fn test_list_owned_txouts() {
         }],
         output: vec![TxOut {
             value: 10000,
-            script_pubkey: trusted_spks[1].clone(),
+            script_pubkey: trusted_spks[1].to_owned(),
         }],
         ..common::new_tx(0)
     };
@@ -186,7 +187,7 @@ fn test_list_owned_txouts() {
     let tx4 = Transaction {
         output: vec![TxOut {
             value: 20000,
-            script_pubkey: untrusted_spks[1].clone(),
+            script_pubkey: untrusted_spks[1].to_owned(),
         }],
         ..common::new_tx(0)
     };
@@ -195,7 +196,7 @@ fn test_list_owned_txouts() {
     let tx5 = Transaction {
         output: vec![TxOut {
             value: 15000,
-            script_pubkey: trusted_spks[2].clone(),
+            script_pubkey: trusted_spks[2].to_owned(),
         }],
         ..common::new_tx(0)
     };
@@ -206,25 +207,24 @@ fn test_list_owned_txouts() {
     // Insert transactions into graph with respective anchors
     // For unconfirmed txs we pass in `None`.
 
-    let _ = graph.insert_relevant_txs(
-        [&tx1, &tx2, &tx3, &tx6].iter().enumerate().map(|(i, tx)| {
+    let _ =
+        graph.batch_insert_relevant([&tx1, &tx2, &tx3, &tx6].iter().enumerate().map(|(i, tx)| {
             let height = i as u32;
             (
                 *tx,
                 local_chain
                     .blocks()
                     .get(&height)
-                    .map(|&hash| BlockId { height, hash })
+                    .cloned()
+                    .map(|hash| BlockId { height, hash })
                     .map(|anchor_block| ConfirmationHeightAnchor {
                         anchor_block,
                         confirmation_height: anchor_block.height,
                     }),
             )
-        }),
-        None,
-    );
+        }));
 
-    let _ = graph.insert_relevant_txs([&tx4, &tx5].iter().map(|tx| (*tx, None)), Some(100));
+    let _ = graph.batch_insert_relevant_unconfirmed([&tx4, &tx5].iter().map(|tx| (*tx, 100)));
 
     // A helper lambda to extract and filter data from the graph.
     let fetch =
@@ -234,7 +234,7 @@ fn test_list_owned_txouts() {
                 .blocks()
                 .get(&height)
                 .map(|&hash| BlockId { height, hash })
-                .expect("block must exist");
+                .unwrap_or_else(|| panic!("block must exist at {}", height));
             let txouts = graph
                 .graph()
                 .filter_chain_txouts(
@@ -257,7 +257,7 @@ fn test_list_owned_txouts() {
                 &local_chain,
                 chain_tip,
                 graph.index.outpoints().iter().cloned(),
-                |_, spk: &Script| trusted_spks.contains(spk),
+                |_, spk: &Script| trusted_spks.contains(&spk.to_owned()),
             );
 
             assert_eq!(txouts.len(), 5);

crates/chain/tests/test_keychain_txout_index.rs

@@ -4,10 +4,12 @@
 mod common;
 use bdk_chain::{
     collections::BTreeMap,
-    keychain::{DerivationAdditions, KeychainTxOutIndex},
+    indexed_tx_graph::Indexer,
+    keychain::{self, KeychainTxOutIndex},
+    Append,
 };
 
-use bitcoin::{secp256k1::Secp256k1, OutPoint, Script, Transaction, TxOut};
+use bitcoin::{secp256k1::Secp256k1, OutPoint, ScriptBuf, Transaction, TxOut};
 use miniscript::{Descriptor, DescriptorPublicKey};
 
 #[derive(Clone, Debug, PartialEq, Eq, Ord, PartialOrd)]
@@ -16,12 +18,14 @@ enum TestKeychain {
     Internal,
 }
 
-fn init_txout_index() -> (
+fn init_txout_index(
+    lookahead: u32,
+) -> (
     bdk_chain::keychain::KeychainTxOutIndex<TestKeychain>,
     Descriptor<DescriptorPublicKey>,
     Descriptor<DescriptorPublicKey>,
 ) {
-    let mut txout_index = bdk_chain::keychain::KeychainTxOutIndex::<TestKeychain>::default();
+    let mut txout_index = bdk_chain::keychain::KeychainTxOutIndex::<TestKeychain>::new(lookahead);
 
     let secp = bdk_chain::bitcoin::secp256k1::Secp256k1::signing_only();
     let (external_descriptor,_) = Descriptor::<DescriptorPublicKey>::parse_descriptor(&secp, "tr([73c5da0a/86'/0'/0']xprv9xgqHN7yz9MwCkxsBPN5qetuNdQSUttZNKw1dcYTV4mkaAFiBVGQziHs3NRSWMkCzvgjEe3n9xV8oYywvM8at9yRqyaZVz6TYYhX98VjsUk/0/*)").unwrap();
@@ -33,7 +37,7 @@ fn init_txout_index() -> (
     (txout_index, external_descriptor, internal_descriptor)
 }
 
-fn spk_at_index(descriptor: &Descriptor<DescriptorPublicKey>, index: u32) -> Script {
+fn spk_at_index(descriptor: &Descriptor<DescriptorPublicKey>, index: u32) -> ScriptBuf {
     descriptor
         .derived_descriptor(&Secp256k1::verification_only(), index)
         .expect("must derive")
@@ -42,7 +46,9 @@ fn spk_at_index(descriptor: &Descriptor<DescriptorPublicKey>, index: u32) -> Scr
 
 #[test]
 fn test_set_all_derivation_indices() {
-    let (mut txout_index, _, _) = init_txout_index();
+    use bdk_chain::indexed_tx_graph::Indexer;
+
+    let (mut txout_index, _, _) = init_txout_index(0);
     let derive_to: BTreeMap<_, _> =
         [(TestKeychain::External, 12), (TestKeychain::Internal, 24)].into();
     assert_eq!(
@@ -52,83 +58,75 @@ fn test_set_all_derivation_indices() {
     assert_eq!(txout_index.last_revealed_indices(), &derive_to);
     assert_eq!(
         txout_index.reveal_to_target_multi(&derive_to).1,
-        DerivationAdditions::default(),
+        keychain::ChangeSet::default(),
         "no changes if we set to the same thing"
     );
+    assert_eq!(txout_index.initial_changeset().as_inner(), &derive_to);
 }
 
 #[test]
 fn test_lookahead() {
-    let (mut txout_index, external_desc, internal_desc) = init_txout_index();
-
-    // ensure it does not break anything if lookahead is set multiple times
-    (0..=10).for_each(|lookahead| txout_index.set_lookahead(&TestKeychain::External, lookahead));
-    (0..=20)
-        .filter(|v| v % 2 == 0)
-        .for_each(|lookahead| txout_index.set_lookahead(&TestKeychain::Internal, lookahead));
-
-    assert_eq!(txout_index.inner().all_spks().len(), 30);
+    let (mut txout_index, external_desc, internal_desc) = init_txout_index(10);
 
     // given:
     // - external lookahead set to 10
-    // - internal lookahead set to 20
     // when:
     // - set external derivation index to value higher than last, but within the lookahead value
     // expect:
     // - scripts cached in spk_txout_index should increase correctly
     // - stored scripts of external keychain should be of expected counts
     for index in (0..20).skip_while(|i| i % 2 == 1) {
-        let (revealed_spks, revealed_additions) =
+        let (revealed_spks, revealed_changeset) =
             txout_index.reveal_to_target(&TestKeychain::External, index);
         assert_eq!(
             revealed_spks.collect::<Vec<_>>(),
             vec![(index, spk_at_index(&external_desc, index))],
         );
         assert_eq!(
-            revealed_additions.as_inner(),
+            revealed_changeset.as_inner(),
             &[(TestKeychain::External, index)].into()
         );
 
         assert_eq!(
             txout_index.inner().all_spks().len(),
             10 /* external lookahead */ +
-            20 /* internal lookahead */ +
+            10 /* internal lookahead */ +
             index as usize + 1 /* `derived` count */
         );
         assert_eq!(
             txout_index
-                .revealed_spks_of_keychain(&TestKeychain::External)
+                .revealed_keychain_spks(&TestKeychain::External)
                 .count(),
             index as usize + 1,
         );
         assert_eq!(
             txout_index
-                .revealed_spks_of_keychain(&TestKeychain::Internal)
+                .revealed_keychain_spks(&TestKeychain::Internal)
                 .count(),
             0,
         );
         assert_eq!(
             txout_index
-                .unused_spks_of_keychain(&TestKeychain::External)
+                .unused_keychain_spks(&TestKeychain::External)
                 .count(),
             index as usize + 1,
         );
         assert_eq!(
             txout_index
-                .unused_spks_of_keychain(&TestKeychain::Internal)
+                .unused_keychain_spks(&TestKeychain::Internal)
                 .count(),
             0,
         );
     }
 
     // given:
-    // - internal lookahead is 20
+    // - internal lookahead is 10
     // - internal derivation index is `None`
     // when:
     // - derivation index is set ahead of current derivation index + lookahead
     // expect:
     // - scripts cached in spk_txout_index should increase correctly, a.k.a. no scripts are skipped
-    let (revealed_spks, revealed_additions) =
+    let (revealed_spks, revealed_changeset) =
         txout_index.reveal_to_target(&TestKeychain::Internal, 24);
     assert_eq!(
         revealed_spks.collect::<Vec<_>>(),
@@ -137,19 +135,19 @@ fn test_lookahead() {
             .collect::<Vec<_>>(),
     );
     assert_eq!(
-        revealed_additions.as_inner(),
+        revealed_changeset.as_inner(),
         &[(TestKeychain::Internal, 24)].into()
     );
     assert_eq!(
         txout_index.inner().all_spks().len(),
         10 /* external lookahead */ +
-        20 /* internal lookahead */ +
+        10 /* internal lookahead */ +
         20 /* external stored index count */ +
         25 /* internal stored index count */
     );
     assert_eq!(
         txout_index
-            .revealed_spks_of_keychain(&TestKeychain::Internal)
+            .revealed_keychain_spks(&TestKeychain::Internal)
             .count(),
         25,
     );
@@ -176,19 +174,21 @@ fn test_lookahead() {
                 TxOut {
                     script_pubkey: external_desc
                         .at_derivation_index(external_index)
+                        .unwrap()
                         .script_pubkey(),
                     value: 10_000,
                 },
                 TxOut {
                     script_pubkey: internal_desc
                         .at_derivation_index(internal_index)
+                        .unwrap()
                         .script_pubkey(),
                     value: 10_000,
                 },
             ],
             ..common::new_tx(external_index)
         };
-        assert_eq!(txout_index.scan(&tx), DerivationAdditions::default());
+        assert_eq!(txout_index.index_tx(&tx), keychain::ChangeSet::default());
         assert_eq!(
             txout_index.last_revealed_index(&TestKeychain::External),
             Some(last_external_index)
@@ -199,13 +199,13 @@ fn test_lookahead() {
         );
         assert_eq!(
             txout_index
-                .revealed_spks_of_keychain(&TestKeychain::External)
+                .revealed_keychain_spks(&TestKeychain::External)
                 .count(),
             last_external_index as usize + 1,
         );
         assert_eq!(
             txout_index
-                .revealed_spks_of_keychain(&TestKeychain::Internal)
+                .revealed_keychain_spks(&TestKeychain::Internal)
                 .count(),
             last_internal_index as usize + 1,
         );
@@ -219,12 +219,19 @@ fn test_lookahead() {
 // - last used index should change as expected
 #[test]
 fn test_scan_with_lookahead() {
-    let (mut txout_index, external_desc, _) = init_txout_index();
-    txout_index.set_lookahead_for_all(10);
+    let (mut txout_index, external_desc, _) = init_txout_index(10);
 
-    let spks: BTreeMap<u32, Script> = [0, 10, 20, 30]
+    let spks: BTreeMap<u32, ScriptBuf> = [0, 10, 20, 30]
         .into_iter()
-        .map(|i| (i, external_desc.at_derivation_index(i).script_pubkey()))
+        .map(|i| {
+            (
+                i,
+                external_desc
+                    .at_derivation_index(i)
+                    .unwrap()
+                    .script_pubkey(),
+            )
+        })
         .collect();
 
     for (&spk_i, spk) in &spks {
@@ -234,9 +241,9 @@ fn test_scan_with_lookahead() {
             value: 0,
         };
 
-        let additions = txout_index.scan_txout(op, &txout);
+        let changeset = txout_index.index_txout(op, &txout);
         assert_eq!(
-            additions.as_inner(),
+            changeset.as_inner(),
             &[(TestKeychain::External, spk_i)].into()
         );
         assert_eq!(
@@ -250,36 +257,40 @@ fn test_scan_with_lookahead() {
     }
 
     // now try with index 41 (lookahead surpassed), we expect that the txout to not be indexed
-    let spk_41 = external_desc.at_derivation_index(41).script_pubkey();
+    let spk_41 = external_desc
+        .at_derivation_index(41)
+        .unwrap()
+        .script_pubkey();
     let op = OutPoint::new(h!("fake tx"), 41);
     let txout = TxOut {
         script_pubkey: spk_41,
         value: 0,
     };
-    let additions = txout_index.scan_txout(op, &txout);
-    assert!(additions.is_empty());
+    let changeset = txout_index.index_txout(op, &txout);
+    assert!(changeset.is_empty());
 }
 
 #[test]
+#[rustfmt::skip]
 fn test_wildcard_derivations() {
-    let (mut txout_index, external_desc, _) = init_txout_index();
-    let external_spk_0 = external_desc.at_derivation_index(0).script_pubkey();
-    let external_spk_16 = external_desc.at_derivation_index(16).script_pubkey();
-    let external_spk_26 = external_desc.at_derivation_index(26).script_pubkey();
-    let external_spk_27 = external_desc.at_derivation_index(27).script_pubkey();
+    let (mut txout_index, external_desc, _) = init_txout_index(0);
+    let external_spk_0 = external_desc.at_derivation_index(0).unwrap().script_pubkey();
+    let external_spk_16 = external_desc.at_derivation_index(16).unwrap().script_pubkey();
+    let external_spk_26 = external_desc.at_derivation_index(26).unwrap().script_pubkey();
+    let external_spk_27 = external_desc.at_derivation_index(27).unwrap().script_pubkey();
 
     // - nothing is derived
     // - unused list is also empty
     //
     // - next_derivation_index() == (0, true)
-    // - derive_new() == ((0, <spk>), DerivationAdditions)
-    // - next_unused() == ((0, <spk>), DerivationAdditions:is_empty())
+    // - derive_new() == ((0, <spk>), keychain::ChangeSet)
+    // - next_unused() == ((0, <spk>), keychain::ChangeSet:is_empty())
     assert_eq!(txout_index.next_index(&TestKeychain::External), (0, true));
     let (spk, changeset) = txout_index.reveal_next_spk(&TestKeychain::External);
-    assert_eq!(spk, (0_u32, &external_spk_0));
+    assert_eq!(spk, (0_u32, external_spk_0.as_script()));
     assert_eq!(changeset.as_inner(), &[(TestKeychain::External, 0)].into());
     let (spk, changeset) = txout_index.next_unused_spk(&TestKeychain::External);
-    assert_eq!(spk, (0_u32, &external_spk_0));
+    assert_eq!(spk, (0_u32, external_spk_0.as_script()));
     assert_eq!(changeset.as_inner(), &[].into());
 
     // - derived till 25
@@ -288,44 +299,45 @@ fn test_wildcard_derivations() {
     // - unused list: [16, 18, 19, 21, 22, 24, 25]
 
     // - next_derivation_index() = (26, true)
-    // - derive_new() = ((26, <spk>), DerivationAdditions)
-    // - next_unused() == ((16, <spk>), DerivationAdditions::is_empty())
+    // - derive_new() = ((26, <spk>), keychain::ChangeSet)
+    // - next_unused() == ((16, <spk>), keychain::ChangeSet::is_empty())
     let _ = txout_index.reveal_to_target(&TestKeychain::External, 25);
 
     (0..=15)
-        .chain(vec![17, 20, 23].into_iter())
-        .for_each(|index| assert!(txout_index.mark_used(&TestKeychain::External, index)));
+        .chain([17, 20, 23])
+        .for_each(|index| assert!(txout_index.mark_used(TestKeychain::External, index)));
 
     assert_eq!(txout_index.next_index(&TestKeychain::External), (26, true));
 
     let (spk, changeset) = txout_index.reveal_next_spk(&TestKeychain::External);
-    assert_eq!(spk, (26, &external_spk_26));
+    assert_eq!(spk, (26, external_spk_26.as_script()));
 
     assert_eq!(changeset.as_inner(), &[(TestKeychain::External, 26)].into());
 
     let (spk, changeset) = txout_index.next_unused_spk(&TestKeychain::External);
-    assert_eq!(spk, (16, &external_spk_16));
+    assert_eq!(spk, (16, external_spk_16.as_script()));
     assert_eq!(changeset.as_inner(), &[].into());
 
     // - Use all the derived till 26.
-    // - next_unused() = ((27, <spk>), DerivationAdditions)
+    // - next_unused() = ((27, <spk>), keychain::ChangeSet)
     (0..=26).for_each(|index| {
-        txout_index.mark_used(&TestKeychain::External, index);
+        txout_index.mark_used(TestKeychain::External, index);
     });
 
     let (spk, changeset) = txout_index.next_unused_spk(&TestKeychain::External);
-    assert_eq!(spk, (27, &external_spk_27));
+    assert_eq!(spk, (27, external_spk_27.as_script()));
     assert_eq!(changeset.as_inner(), &[(TestKeychain::External, 27)].into());
 }
 
 #[test]
 fn test_non_wildcard_derivations() {
-    let mut txout_index = KeychainTxOutIndex::<TestKeychain>::default();
+    let mut txout_index = KeychainTxOutIndex::<TestKeychain>::new(0);
 
     let secp = bitcoin::secp256k1::Secp256k1::signing_only();
     let (no_wildcard_descriptor, _) = Descriptor::<DescriptorPublicKey>::parse_descriptor(&secp, "wpkh([73c5da0a/86'/0'/0']xprv9xgqHN7yz9MwCkxsBPN5qetuNdQSUttZNKw1dcYTV4mkaAFiBVGQziHs3NRSWMkCzvgjEe3n9xV8oYywvM8at9yRqyaZVz6TYYhX98VjsUk/1/0)").unwrap();
     let external_spk = no_wildcard_descriptor
         .at_derivation_index(0)
+        .unwrap()
         .script_pubkey();
 
     txout_index.add_keychain(TestKeychain::External, no_wildcard_descriptor);
@@ -338,11 +350,11 @@ fn test_non_wildcard_derivations() {
     // - when we get the next unused script, script @ index 0
     assert_eq!(txout_index.next_index(&TestKeychain::External), (0, true));
     let (spk, changeset) = txout_index.reveal_next_spk(&TestKeychain::External);
-    assert_eq!(spk, (0, &external_spk));
+    assert_eq!(spk, (0, external_spk.as_script()));
     assert_eq!(changeset.as_inner(), &[(TestKeychain::External, 0)].into());
 
     let (spk, changeset) = txout_index.next_unused_spk(&TestKeychain::External);
-    assert_eq!(spk, (0, &external_spk));
+    assert_eq!(spk, (0, external_spk.as_script()));
     assert_eq!(changeset.as_inner(), &[].into());
 
     // given:
@@ -350,19 +362,127 @@ fn test_non_wildcard_derivations() {
     // expect:
     // - next derivation index should not be new
     // - derive new and next unused should return the old script
-    // - store_up_to should not panic and return empty additions
+    // - store_up_to should not panic and return empty changeset
     assert_eq!(txout_index.next_index(&TestKeychain::External), (0, false));
-    txout_index.mark_used(&TestKeychain::External, 0);
+    txout_index.mark_used(TestKeychain::External, 0);
 
     let (spk, changeset) = txout_index.reveal_next_spk(&TestKeychain::External);
-    assert_eq!(spk, (0, &external_spk));
+    assert_eq!(spk, (0, external_spk.as_script()));
     assert_eq!(changeset.as_inner(), &[].into());
 
     let (spk, changeset) = txout_index.next_unused_spk(&TestKeychain::External);
-    assert_eq!(spk, (0, &external_spk));
+    assert_eq!(spk, (0, external_spk.as_script()));
     assert_eq!(changeset.as_inner(), &[].into());
-    let (revealed_spks, revealed_additions) =
+    let (revealed_spks, revealed_changeset) =
         txout_index.reveal_to_target(&TestKeychain::External, 200);
     assert_eq!(revealed_spks.count(), 0);
-    assert!(revealed_additions.is_empty());
+    assert!(revealed_changeset.is_empty());
+
+    // we check that spks_of_keychain returns a SpkIterator with just one element
+    assert_eq!(
+        txout_index
+            .revealed_keychain_spks(&TestKeychain::External)
+            .count(),
+        1,
+    );
+}
+
+/// Check that calling `lookahead_to_target` stores the expected spks.
+#[test]
+fn lookahead_to_target() {
+    #[derive(Default)]
+    struct TestCase {
+        /// Global lookahead value.
+        lookahead: u32,
+        /// Last revealed index for external keychain.
+        external_last_revealed: Option<u32>,
+        /// Last revealed index for internal keychain.
+        internal_last_revealed: Option<u32>,
+        /// Call `lookahead_to_target(External, u32)`.
+        external_target: Option<u32>,
+        /// Call `lookahead_to_target(Internal, u32)`.
+        internal_target: Option<u32>,
+    }
+
+    let test_cases = &[
+        TestCase {
+            lookahead: 0,
+            external_target: Some(100),
+            ..Default::default()
+        },
+        TestCase {
+            lookahead: 10,
+            internal_target: Some(99),
+            ..Default::default()
+        },
+        TestCase {
+            lookahead: 100,
+            internal_target: Some(9),
+            external_target: Some(10),
+            ..Default::default()
+        },
+        TestCase {
+            lookahead: 12,
+            external_last_revealed: Some(2),
+            internal_last_revealed: Some(2),
+            internal_target: Some(15),
+            external_target: Some(13),
+        },
+        TestCase {
+            lookahead: 13,
+            external_last_revealed: Some(100),
+            internal_last_revealed: Some(21),
+            internal_target: Some(120),
+            external_target: Some(130),
+        },
+    ];
+
+    for t in test_cases {
+        let (mut index, _, _) = init_txout_index(t.lookahead);
+
+        if let Some(last_revealed) = t.external_last_revealed {
+            let _ = index.reveal_to_target(&TestKeychain::External, last_revealed);
+        }
+        if let Some(last_revealed) = t.internal_last_revealed {
+            let _ = index.reveal_to_target(&TestKeychain::Internal, last_revealed);
+        }
+
+        let keychain_test_cases = [
+            (
+                TestKeychain::External,
+                t.external_last_revealed,
+                t.external_target,
+            ),
+            (
+                TestKeychain::Internal,
+                t.internal_last_revealed,
+                t.internal_target,
+            ),
+        ];
+        for (keychain, last_revealed, target) in keychain_test_cases {
+            if let Some(target) = target {
+                let original_last_stored_index = match last_revealed {
+                    Some(last_revealed) => Some(last_revealed + t.lookahead),
+                    None => t.lookahead.checked_sub(1),
+                };
+                let exp_last_stored_index = match original_last_stored_index {
+                    Some(original_last_stored_index) => {
+                        Ord::max(target, original_last_stored_index)
+                    }
+                    None => target,
+                };
+                index.lookahead_to_target(&keychain, target);
+                let keys = index
+                    .inner()
+                    .all_spks()
+                    .range((keychain.clone(), 0)..=(keychain.clone(), u32::MAX))
+                    .map(|(k, _)| k.clone())
+                    .collect::<Vec<_>>();
+                let exp_keys = core::iter::repeat(keychain)
+                    .zip(0_u32..=exp_last_stored_index)
+                    .collect::<Vec<_>>();
+                assert_eq!(keys, exp_keys);
+            }
+        }
+    }
 }

crates/chain/tests/test_local_chain.rs

@@ -1,217 +1,366 @@
-use bdk_chain::local_chain::{
-    ChangeSet, InsertBlockNotMatchingError, LocalChain, UpdateNotConnectedError,
+use bdk_chain::{
+    local_chain::{
+        AlterCheckPointError, ApplyHeaderError, CannotConnectError, ChangeSet, CheckPoint,
+        LocalChain, MissingGenesisError, Update,
+    },
+    BlockId,
 };
-use bitcoin::BlockHash;
+use bitcoin::{block::Header, hashes::Hash, BlockHash};
 
 #[macro_use]
 mod common;
 
-#[test]
-fn add_first_tip() {
-    let chain = LocalChain::default();
-    assert_eq!(
-        chain.determine_changeset(&local_chain![(0, h!("A"))]),
-        Ok([(0, Some(h!("A")))].into()),
-        "add first tip"
-    );
+#[derive(Debug)]
+struct TestLocalChain<'a> {
+    name: &'static str,
+    chain: LocalChain,
+    update: Update,
+    exp: ExpectedResult<'a>,
+}
+
+#[derive(Debug, PartialEq)]
+enum ExpectedResult<'a> {
+    Ok {
+        changeset: &'a [(u32, Option<BlockHash>)],
+        init_changeset: &'a [(u32, Option<BlockHash>)],
+    },
+    Err(CannotConnectError),
+}
+
+impl<'a> TestLocalChain<'a> {
+    fn run(mut self) {
+        println!("[TestLocalChain] test: {}", self.name);
+        let got_changeset = match self.chain.apply_update(self.update) {
+            Ok(changeset) => changeset,
+            Err(got_err) => {
+                assert_eq!(
+                    ExpectedResult::Err(got_err),
+                    self.exp,
+                    "{}: unexpected error",
+                    self.name
+                );
+                return;
+            }
+        };
+
+        match self.exp {
+            ExpectedResult::Ok {
+                changeset,
+                init_changeset,
+            } => {
+                assert_eq!(
+                    got_changeset,
+                    changeset.iter().cloned().collect(),
+                    "{}: unexpected changeset",
+                    self.name
+                );
+                assert_eq!(
+                    self.chain.initial_changeset(),
+                    init_changeset.iter().cloned().collect(),
+                    "{}: unexpected initial changeset",
+                    self.name
+                );
+            }
+            ExpectedResult::Err(err) => panic!(
+                "{}: expected error ({}), got non-error result: {:?}",
+                self.name, err, got_changeset
+            ),
+        }
+    }
 }
 
 #[test]
-fn add_second_tip() {
-    let chain = local_chain![(0, h!("A"))];
-    assert_eq!(
-        chain.determine_changeset(&local_chain![(0, h!("A")), (1, h!("B"))]),
-        Ok([(1, Some(h!("B")))].into())
-    );
+fn update_local_chain() {
+    [
+        TestLocalChain {
+            name: "add first tip",
+            chain: local_chain![(0, h!("A"))],
+            update: chain_update![(0, h!("A"))],
+            exp: ExpectedResult::Ok {
+                changeset: &[],
+                init_changeset: &[(0, Some(h!("A")))],
+            },
+        },
+        TestLocalChain {
+            name: "add second tip",
+            chain: local_chain![(0, h!("A"))],
+            update: chain_update![(0, h!("A")), (1, h!("B"))],
+            exp: ExpectedResult::Ok {
+                changeset: &[(1, Some(h!("B")))],
+                init_changeset: &[(0, Some(h!("A"))), (1, Some(h!("B")))],
+            },
+        },
+        TestLocalChain {
+            name: "two disjoint chains cannot merge",
+            chain: local_chain![(0, h!("_")), (1, h!("A"))],
+            update: chain_update![(0, h!("_")), (2, h!("B"))],
+            exp: ExpectedResult::Err(CannotConnectError {
+                try_include_height: 1,
+            }),
+        },
+        TestLocalChain {
+            name: "two disjoint chains cannot merge (existing chain longer)",
+            chain: local_chain![(0, h!("_")), (2, h!("A"))],
+            update: chain_update![(0, h!("_")), (1, h!("B"))],
+            exp: ExpectedResult::Err(CannotConnectError {
+                try_include_height: 2,
+            }),
+        },
+        TestLocalChain {
+            name: "duplicate chains should merge",
+            chain: local_chain![(0, h!("A"))],
+            update: chain_update![(0, h!("A"))],
+            exp: ExpectedResult::Ok {
+                changeset: &[],
+                init_changeset: &[(0, Some(h!("A")))],
+            },
+        },
+        // Introduce an older checkpoint (B)
+        //        | 0 | 1 | 2 | 3
+        // chain  | _       C   D
+        // update | _   B   C
+        TestLocalChain {
+            name: "can introduce older checkpoint",
+            chain: local_chain![(0, h!("_")), (2, h!("C")), (3, h!("D"))],
+            update: chain_update![(0, h!("_")), (1, h!("B")), (2, h!("C"))],
+            exp: ExpectedResult::Ok {
+                changeset: &[(1, Some(h!("B")))],
+                init_changeset: &[(0, Some(h!("_"))), (1, Some(h!("B"))), (2, Some(h!("C"))), (3, Some(h!("D")))],
+            },
+        },
+        // Introduce an older checkpoint (A) that is not directly behind PoA
+        //        | 0 | 2 | 3 | 4
+        // chain  | _       B   C
+        // update | _   A       C
+        TestLocalChain {
+            name: "can introduce older checkpoint 2",
+            chain: local_chain![(0, h!("_")), (3, h!("B")), (4, h!("C"))],
+            update: chain_update![(0, h!("_")), (2, h!("A")), (4, h!("C"))],
+            exp: ExpectedResult::Ok {
+                changeset: &[(2, Some(h!("A")))],
+                init_changeset: &[(0, Some(h!("_"))), (2, Some(h!("A"))), (3, Some(h!("B"))), (4, Some(h!("C")))],
+            }
+        },
+        // Introduce an older checkpoint (B) that is not the oldest checkpoint
+        //        | 0 | 1 | 2 | 3
+        // chain  | _   A       C
+        // update | _       B   C
+        TestLocalChain {
+            name: "can introduce older checkpoint 3",
+            chain: local_chain![(0, h!("_")), (1, h!("A")), (3, h!("C"))],
+            update: chain_update![(0, h!("_")), (2, h!("B")), (3, h!("C"))],
+            exp: ExpectedResult::Ok {
+                changeset: &[(2, Some(h!("B")))],
+                init_changeset: &[(0, Some(h!("_"))), (1, Some(h!("A"))), (2, Some(h!("B"))), (3, Some(h!("C")))],
+            }
+        },
+        // Introduce two older checkpoints below the PoA
+        //        | 0 | 1 | 2 | 3
+        // chain  | _           C
+        // update | _   A   B   C
+        TestLocalChain {
+            name: "introduce two older checkpoints below PoA",
+            chain: local_chain![(0, h!("_")), (3, h!("C"))],
+            update: chain_update![(0, h!("_")), (1, h!("A")), (2, h!("B")), (3, h!("C"))],
+            exp: ExpectedResult::Ok {
+                changeset: &[(1, Some(h!("A"))), (2, Some(h!("B")))],
+                init_changeset: &[(0, Some(h!("_"))), (1, Some(h!("A"))), (2, Some(h!("B"))), (3, Some(h!("C")))],
+            },
+        },
+        TestLocalChain {
+            name: "fix blockhash before agreement point",
+            chain: local_chain![(0, h!("im-wrong")), (1, h!("we-agree"))],
+            update: chain_update![(0, h!("fix")), (1, h!("we-agree"))],
+            exp: ExpectedResult::Ok {
+                changeset: &[(0, Some(h!("fix")))],
+                init_changeset: &[(0, Some(h!("fix"))), (1, Some(h!("we-agree")))],
+            },
+        },
+        // B and C are in both chain and update
+        //        | 0 | 1 | 2 | 3 | 4
+        // chain  | _       B   C
+        // update | _   A   B   C   D
+        // This should succeed with the point of agreement being C and A should be added in addition.
+        TestLocalChain {
+            name: "two points of agreement",
+            chain: local_chain![(0, h!("_")), (2, h!("B")), (3, h!("C"))],
+            update: chain_update![(0, h!("_")), (1, h!("A")), (2, h!("B")), (3, h!("C")), (4, h!("D"))],
+            exp: ExpectedResult::Ok {
+                changeset: &[(1, Some(h!("A"))), (4, Some(h!("D")))],
+                init_changeset: &[
+                    (0, Some(h!("_"))),
+                    (1, Some(h!("A"))),
+                    (2, Some(h!("B"))),
+                    (3, Some(h!("C"))),
+                    (4, Some(h!("D"))),
+                ],
+            },
+        },
+        // Update and chain does not connect:
+        //        | 0 | 1 | 2 | 3 | 4
+        // chain  | _       B   C
+        // update | _   A   B       D
+        // This should fail as we cannot figure out whether C & D are on the same chain
+        TestLocalChain {
+            name: "update and chain does not connect",
+            chain: local_chain![(0, h!("_")), (2, h!("B")), (3, h!("C"))],
+            update: chain_update![(0, h!("_")), (1, h!("A")), (2, h!("B")), (4, h!("D"))],
+            exp: ExpectedResult::Err(CannotConnectError {
+                try_include_height: 3,
+            }),
+        },
+        // Transient invalidation:
+        //        | 0 | 1 | 2 | 3 | 4 | 5
+        // chain  | _       B   C       E
+        // update | _       B'  C'  D
+        // This should succeed and invalidate B,C and E with point of agreement being A.
+        TestLocalChain {
+            name: "transitive invalidation applies to checkpoints higher than invalidation",
+            chain: local_chain![(0, h!("_")), (2, h!("B")), (3, h!("C")), (5, h!("E"))],
+            update: chain_update![(0, h!("_")), (2, h!("B'")), (3, h!("C'")), (4, h!("D"))],
+            exp: ExpectedResult::Ok {
+                changeset: &[
+                    (2, Some(h!("B'"))),
+                    (3, Some(h!("C'"))),
+                    (4, Some(h!("D"))),
+                    (5, None),
+                ],
+                init_changeset: &[
+                    (0, Some(h!("_"))),
+                    (2, Some(h!("B'"))),
+                    (3, Some(h!("C'"))),
+                    (4, Some(h!("D"))),
+                ],
+            },
+        },
+        // Transient invalidation:
+        //        | 0 | 1 | 2 | 3 | 4
+        // chain  | _   B   C       E
+        // update | _   B'  C'  D
+        // This should succeed and invalidate B, C and E with no point of agreement
+        TestLocalChain {
+            name: "transitive invalidation applies to checkpoints higher than invalidation no point of agreement",
+            chain: local_chain![(0, h!("_")), (1, h!("B")), (2, h!("C")), (4, h!("E"))],
+            update: chain_update![(0, h!("_")), (1, h!("B'")), (2, h!("C'")), (3, h!("D"))],
+            exp: ExpectedResult::Ok {
+                changeset: &[
+                    (1, Some(h!("B'"))),
+                    (2, Some(h!("C'"))),
+                    (3, Some(h!("D"))),
+                    (4, None)
+                ],
+                init_changeset: &[
+                    (0, Some(h!("_"))), 
+                    (1, Some(h!("B'"))),
+                    (2, Some(h!("C'"))),
+                    (3, Some(h!("D"))),
+                ],
+            },
+        },
+        // Transient invalidation:
+        //        | 0 | 1 | 2 | 3 | 4 | 5
+        // chain  | _   A   B   C       E
+        // update | _       B'  C'  D
+        // This should fail since although it tells us that B and C are invalid it doesn't tell us whether
+        // A was invalid.
+        TestLocalChain {
+            name: "invalidation but no connection",
+            chain: local_chain![(0, h!("_")), (1, h!("A")), (2, h!("B")), (3, h!("C")), (5, h!("E"))],
+            update: chain_update![(0, h!("_")), (2, h!("B'")), (3, h!("C'")), (4, h!("D"))],
+            exp: ExpectedResult::Err(CannotConnectError { try_include_height: 1 }),
+        },
+        // Introduce blocks between two points of agreement
+        //        | 0 | 1 | 2 | 3 | 4 | 5
+        // chain  | A   B       D   E
+        // update | A       C       E   F
+        TestLocalChain {
+            name: "introduce blocks between two points of agreement",
+            chain: local_chain![(0, h!("A")), (1, h!("B")), (3, h!("D")), (4, h!("E"))],
+            update: chain_update![(0, h!("A")), (2, h!("C")), (4, h!("E")), (5, h!("F"))],
+            exp: ExpectedResult::Ok {
+                changeset: &[
+                    (2, Some(h!("C"))),
+                    (5, Some(h!("F"))),
+                ],
+                init_changeset: &[
+                    (0, Some(h!("A"))),
+                    (1, Some(h!("B"))),
+                    (2, Some(h!("C"))),
+                    (3, Some(h!("D"))),
+                    (4, Some(h!("E"))),
+                    (5, Some(h!("F"))),
+                ],
+            },
+        },
+        // Allow update that is shorter than original chain
+        //        | 0 | 1 | 2 | 3 | 4 | 5
+        // chain  | A       C   D   E   F
+        // update | A       C   D'
+        TestLocalChain {
+            name: "allow update that is shorter than original chain",
+            chain: local_chain![(0, h!("_")), (2, h!("C")), (3, h!("D")), (4, h!("E")), (5, h!("F"))],
+            update: chain_update![(0, h!("_")), (2, h!("C")), (3, h!("D'"))],
+            exp: ExpectedResult::Ok {
+                changeset: &[
+                    (3, Some(h!("D'"))),
+                    (4, None),
+                    (5, None),
+                ],
+                init_changeset: &[
+                    (0, Some(h!("_"))),
+                    (2, Some(h!("C"))),
+                    (3, Some(h!("D'"))),
+                ],
+            },
+        },
+    ]
+    .into_iter()
+    .for_each(TestLocalChain::run);
 }
 
 #[test]
-fn two_disjoint_chains_cannot_merge() {
-    let chain1 = local_chain![(0, h!("A"))];
-    let chain2 = local_chain![(1, h!("B"))];
-    assert_eq!(
-        chain1.determine_changeset(&chain2),
-        Err(UpdateNotConnectedError(0))
-    );
-}
-
-#[test]
-fn duplicate_chains_should_merge() {
-    let chain1 = local_chain![(0, h!("A"))];
-    let chain2 = local_chain![(0, h!("A"))];
-    assert_eq!(chain1.determine_changeset(&chain2), Ok(Default::default()));
-}
-
-#[test]
-fn can_introduce_older_checkpoints() {
-    let chain1 = local_chain![(2, h!("C")), (3, h!("D"))];
-    let chain2 = local_chain![(1, h!("B")), (2, h!("C"))];
-
-    assert_eq!(
-        chain1.determine_changeset(&chain2),
-        Ok([(1, Some(h!("B")))].into())
-    );
-}
-
-#[test]
-fn fix_blockhash_before_agreement_point() {
-    let chain1 = local_chain![(0, h!("im-wrong")), (1, h!("we-agree"))];
-    let chain2 = local_chain![(0, h!("fix")), (1, h!("we-agree"))];
-
-    assert_eq!(
-        chain1.determine_changeset(&chain2),
-        Ok([(0, Some(h!("fix")))].into())
-    )
-}
-
-/// B and C are in both chain and update
-/// ```
-///        | 0 | 1 | 2 | 3 | 4
-/// chain  |     B   C
-/// update | A   B   C   D
-/// ```
-/// This should succeed with the point of agreement being C and A should be added in addition.
-#[test]
-fn two_points_of_agreement() {
-    let chain1 = local_chain![(1, h!("B")), (2, h!("C"))];
-    let chain2 = local_chain![(0, h!("A")), (1, h!("B")), (2, h!("C")), (3, h!("D"))];
-
-    assert_eq!(
-        chain1.determine_changeset(&chain2),
-        Ok([(0, Some(h!("A"))), (3, Some(h!("D")))].into()),
-    );
-}
-
-/// Update and chain does not connect:
-/// ```
-///        | 0 | 1 | 2 | 3 | 4
-/// chain  |     B   C
-/// update | A   B       D
-/// ```
-/// This should fail as we cannot figure out whether C & D are on the same chain
-#[test]
-fn update_and_chain_does_not_connect() {
-    let chain1 = local_chain![(1, h!("B")), (2, h!("C"))];
-    let chain2 = local_chain![(0, h!("A")), (1, h!("B")), (3, h!("D"))];
-
-    assert_eq!(
-        chain1.determine_changeset(&chain2),
-        Err(UpdateNotConnectedError(2)),
-    );
-}
-
-/// Transient invalidation:
-/// ```
-///        | 0 | 1 | 2 | 3 | 4 | 5
-/// chain  | A       B   C       E
-/// update | A       B'  C'  D
-/// ```
-/// This should succeed and invalidate B,C and E with point of agreement being A.
-#[test]
-fn transitive_invalidation_applies_to_checkpoints_higher_than_invalidation() {
-    let chain1 = local_chain![(0, h!("A")), (2, h!("B")), (3, h!("C")), (5, h!("E"))];
-    let chain2 = local_chain![(0, h!("A")), (2, h!("B'")), (3, h!("C'")), (4, h!("D"))];
-
-    assert_eq!(
-        chain1.determine_changeset(&chain2),
-        Ok([
-            (2, Some(h!("B'"))),
-            (3, Some(h!("C'"))),
-            (4, Some(h!("D"))),
-            (5, None),
-        ]
-        .into())
-    );
-}
-
-/// Transient invalidation:
-/// ```
-///        | 0 | 1 | 2 | 3 | 4
-/// chain  |     B   C       E
-/// update |     B'  C'  D
-/// ```
-///
-/// This should succeed and invalidate B, C and E with no point of agreement
-#[test]
-fn transitive_invalidation_applies_to_checkpoints_higher_than_invalidation_no_point_of_agreement() {
-    let chain1 = local_chain![(1, h!("B")), (2, h!("C")), (4, h!("E"))];
-    let chain2 = local_chain![(1, h!("B'")), (2, h!("C'")), (3, h!("D"))];
-
-    assert_eq!(
-        chain1.determine_changeset(&chain2),
-        Ok([
-            (1, Some(h!("B'"))),
-            (2, Some(h!("C'"))),
-            (3, Some(h!("D"))),
-            (4, None)
-        ]
-        .into())
-    )
-}
-
-/// Transient invalidation:
-/// ```
-///        | 0 | 1 | 2 | 3 | 4
-/// chain  | A   B   C       E
-/// update |     B'  C'  D
-/// ```
-///
-/// This should fail since although it tells us that B and C are invalid it doesn't tell us whether
-/// A was invalid.
-#[test]
-fn invalidation_but_no_connection() {
-    let chain1 = local_chain![(0, h!("A")), (1, h!("B")), (2, h!("C")), (4, h!("E"))];
-    let chain2 = local_chain![(1, h!("B'")), (2, h!("C'")), (3, h!("D"))];
-
-    assert_eq!(
-        chain1.determine_changeset(&chain2),
-        Err(UpdateNotConnectedError(0))
-    )
-}
-
-#[test]
-fn insert_block() {
+fn local_chain_insert_block() {
     struct TestCase {
         original: LocalChain,
         insert: (u32, BlockHash),
-        expected_result: Result<ChangeSet, InsertBlockNotMatchingError>,
+        expected_result: Result<ChangeSet, AlterCheckPointError>,
         expected_final: LocalChain,
     }
 
     let test_cases = [
         TestCase {
-            original: local_chain![],
+            original: local_chain![(0, h!("_"))],
             insert: (5, h!("block5")),
             expected_result: Ok([(5, Some(h!("block5")))].into()),
-            expected_final: local_chain![(5, h!("block5"))],
+            expected_final: local_chain![(0, h!("_")), (5, h!("block5"))],
         },
         TestCase {
-            original: local_chain![(3, h!("A"))],
+            original: local_chain![(0, h!("_")), (3, h!("A"))],
             insert: (4, h!("B")),
             expected_result: Ok([(4, Some(h!("B")))].into()),
-            expected_final: local_chain![(3, h!("A")), (4, h!("B"))],
+            expected_final: local_chain![(0, h!("_")), (3, h!("A")), (4, h!("B"))],
         },
         TestCase {
-            original: local_chain![(4, h!("B"))],
+            original: local_chain![(0, h!("_")), (4, h!("B"))],
             insert: (3, h!("A")),
             expected_result: Ok([(3, Some(h!("A")))].into()),
-            expected_final: local_chain![(3, h!("A")), (4, h!("B"))],
+            expected_final: local_chain![(0, h!("_")), (3, h!("A")), (4, h!("B"))],
         },
         TestCase {
-            original: local_chain![(2, h!("K"))],
+            original: local_chain![(0, h!("_")), (2, h!("K"))],
             insert: (2, h!("K")),
             expected_result: Ok([].into()),
-            expected_final: local_chain![(2, h!("K"))],
+            expected_final: local_chain![(0, h!("_")), (2, h!("K"))],
         },
         TestCase {
-            original: local_chain![(2, h!("K"))],
+            original: local_chain![(0, h!("_")), (2, h!("K"))],
             insert: (2, h!("J")),
-            expected_result: Err(InsertBlockNotMatchingError {
+            expected_result: Err(AlterCheckPointError {
                 height: 2,
                 original_hash: h!("K"),
-                update_hash: h!("J"),
+                update_hash: Some(h!("J")),
             }),
-            expected_final: local_chain![(2, h!("K"))],
+            expected_final: local_chain![(0, h!("_")), (2, h!("K"))],
         },
     ];
 
@@ -226,3 +375,307 @@ fn insert_block() {
         assert_eq!(chain, t.expected_final, "[{}] unexpected final chain", i,);
     }
 }
+
+#[test]
+fn local_chain_disconnect_from() {
+    struct TestCase {
+        name: &'static str,
+        original: LocalChain,
+        disconnect_from: (u32, BlockHash),
+        exp_result: Result<ChangeSet, MissingGenesisError>,
+        exp_final: LocalChain,
+    }
+
+    let test_cases = [
+        TestCase {
+            name: "try_replace_genesis_should_fail",
+            original: local_chain![(0, h!("_"))],
+            disconnect_from: (0, h!("_")),
+            exp_result: Err(MissingGenesisError),
+            exp_final: local_chain![(0, h!("_"))],
+        },
+        TestCase {
+            name: "try_replace_genesis_should_fail_2",
+            original: local_chain![(0, h!("_")), (2, h!("B")), (3, h!("C"))],
+            disconnect_from: (0, h!("_")),
+            exp_result: Err(MissingGenesisError),
+            exp_final: local_chain![(0, h!("_")), (2, h!("B")), (3, h!("C"))],
+        },
+        TestCase {
+            name: "from_does_not_exist",
+            original: local_chain![(0, h!("_")), (3, h!("C"))],
+            disconnect_from: (2, h!("B")),
+            exp_result: Ok(ChangeSet::default()),
+            exp_final: local_chain![(0, h!("_")), (3, h!("C"))],
+        },
+        TestCase {
+            name: "from_has_different_blockhash",
+            original: local_chain![(0, h!("_")), (2, h!("B"))],
+            disconnect_from: (2, h!("not_B")),
+            exp_result: Ok(ChangeSet::default()),
+            exp_final: local_chain![(0, h!("_")), (2, h!("B"))],
+        },
+        TestCase {
+            name: "disconnect_one",
+            original: local_chain![(0, h!("_")), (2, h!("B"))],
+            disconnect_from: (2, h!("B")),
+            exp_result: Ok(ChangeSet::from_iter([(2, None)])),
+            exp_final: local_chain![(0, h!("_"))],
+        },
+        TestCase {
+            name: "disconnect_three",
+            original: local_chain![(0, h!("_")), (2, h!("B")), (3, h!("C")), (4, h!("D"))],
+            disconnect_from: (2, h!("B")),
+            exp_result: Ok(ChangeSet::from_iter([(2, None), (3, None), (4, None)])),
+            exp_final: local_chain![(0, h!("_"))],
+        },
+    ];
+
+    for (i, t) in test_cases.into_iter().enumerate() {
+        println!("Case {}: {}", i, t.name);
+
+        let mut chain = t.original;
+        let result = chain.disconnect_from(t.disconnect_from.into());
+        assert_eq!(
+            result, t.exp_result,
+            "[{}:{}] unexpected changeset result",
+            i, t.name
+        );
+        assert_eq!(
+            chain, t.exp_final,
+            "[{}:{}] unexpected final chain",
+            i, t.name
+        );
+    }
+}
+
+#[test]
+fn checkpoint_from_block_ids() {
+    struct TestCase<'a> {
+        name: &'a str,
+        blocks: &'a [(u32, BlockHash)],
+        exp_result: Result<(), Option<(u32, BlockHash)>>,
+    }
+
+    let test_cases = [
+        TestCase {
+            name: "in_order",
+            blocks: &[(0, h!("A")), (1, h!("B")), (3, h!("D"))],
+            exp_result: Ok(()),
+        },
+        TestCase {
+            name: "with_duplicates",
+            blocks: &[(1, h!("B")), (2, h!("C")), (2, h!("C'"))],
+            exp_result: Err(Some((2, h!("C")))),
+        },
+        TestCase {
+            name: "not_in_order",
+            blocks: &[(1, h!("B")), (3, h!("D")), (2, h!("C"))],
+            exp_result: Err(Some((3, h!("D")))),
+        },
+        TestCase {
+            name: "empty",
+            blocks: &[],
+            exp_result: Err(None),
+        },
+        TestCase {
+            name: "single",
+            blocks: &[(21, h!("million"))],
+            exp_result: Ok(()),
+        },
+    ];
+
+    for (i, t) in test_cases.into_iter().enumerate() {
+        println!("running test case {}: '{}'", i, t.name);
+        let result = CheckPoint::from_block_ids(
+            t.blocks
+                .iter()
+                .map(|&(height, hash)| BlockId { height, hash }),
+        );
+        match t.exp_result {
+            Ok(_) => {
+                assert!(result.is_ok(), "[{}:{}] should be Ok", i, t.name);
+                let result_vec = {
+                    let mut v = result
+                        .unwrap()
+                        .into_iter()
+                        .map(|cp| (cp.height(), cp.hash()))
+                        .collect::<Vec<_>>();
+                    v.reverse();
+                    v
+                };
+                assert_eq!(
+                    &result_vec, t.blocks,
+                    "[{}:{}] not equal to original block ids",
+                    i, t.name
+                );
+            }
+            Err(exp_last) => {
+                assert!(result.is_err(), "[{}:{}] should be Err", i, t.name);
+                let err = result.unwrap_err();
+                assert_eq!(
+                    err.as_ref()
+                        .map(|last_cp| (last_cp.height(), last_cp.hash())),
+                    exp_last,
+                    "[{}:{}] error's last cp height should be {:?}, got {:?}",
+                    i,
+                    t.name,
+                    exp_last,
+                    err
+                );
+            }
+        }
+    }
+}
+
+#[test]
+fn local_chain_apply_header_connected_to() {
+    fn header_from_prev_blockhash(prev_blockhash: BlockHash) -> Header {
+        Header {
+            version: bitcoin::block::Version::default(),
+            prev_blockhash,
+            merkle_root: bitcoin::hash_types::TxMerkleNode::all_zeros(),
+            time: 0,
+            bits: bitcoin::CompactTarget::default(),
+            nonce: 0,
+        }
+    }
+
+    struct TestCase {
+        name: &'static str,
+        chain: LocalChain,
+        header: Header,
+        height: u32,
+        connected_to: BlockId,
+        exp_result: Result<Vec<(u32, Option<BlockHash>)>, ApplyHeaderError>,
+    }
+
+    let test_cases = [
+        {
+            let header = header_from_prev_blockhash(h!("A"));
+            let hash = header.block_hash();
+            let height = 2;
+            let connected_to = BlockId { height, hash };
+            TestCase {
+                name: "connected_to_self_header_applied_to_self",
+                chain: local_chain![(0, h!("_")), (height, hash)],
+                header,
+                height,
+                connected_to,
+                exp_result: Ok(vec![]),
+            }
+        },
+        {
+            let prev_hash = h!("A");
+            let prev_height = 1;
+            let header = header_from_prev_blockhash(prev_hash);
+            let hash = header.block_hash();
+            let height = prev_height + 1;
+            let connected_to = BlockId {
+                height: prev_height,
+                hash: prev_hash,
+            };
+            TestCase {
+                name: "connected_to_prev_header_applied_to_self",
+                chain: local_chain![(0, h!("_")), (prev_height, prev_hash)],
+                header,
+                height,
+                connected_to,
+                exp_result: Ok(vec![(height, Some(hash))]),
+            }
+        },
+        {
+            let header = header_from_prev_blockhash(BlockHash::all_zeros());
+            let hash = header.block_hash();
+            let height = 0;
+            let connected_to = BlockId { height, hash };
+            TestCase {
+                name: "genesis_applied_to_self",
+                chain: local_chain![(0, hash)],
+                header,
+                height,
+                connected_to,
+                exp_result: Ok(vec![]),
+            }
+        },
+        {
+            let header = header_from_prev_blockhash(h!("Z"));
+            let height = 10;
+            let hash = header.block_hash();
+            let prev_height = height - 1;
+            let prev_hash = header.prev_blockhash;
+            TestCase {
+                name: "connect_at_connected_to",
+                chain: local_chain![(0, h!("_")), (2, h!("B")), (3, h!("C"))],
+                header,
+                height: 10,
+                connected_to: BlockId {
+                    height: 3,
+                    hash: h!("C"),
+                },
+                exp_result: Ok(vec![(prev_height, Some(prev_hash)), (height, Some(hash))]),
+            }
+        },
+        {
+            let prev_hash = h!("A");
+            let prev_height = 1;
+            let header = header_from_prev_blockhash(prev_hash);
+            let connected_to = BlockId {
+                height: prev_height,
+                hash: h!("not_prev_hash"),
+            };
+            TestCase {
+                name: "inconsistent_prev_hash",
+                chain: local_chain![(0, h!("_")), (prev_height, h!("not_prev_hash"))],
+                header,
+                height: prev_height + 1,
+                connected_to,
+                exp_result: Err(ApplyHeaderError::InconsistentBlocks),
+            }
+        },
+        {
+            let prev_hash = h!("A");
+            let prev_height = 1;
+            let header = header_from_prev_blockhash(prev_hash);
+            let height = prev_height + 1;
+            let connected_to = BlockId {
+                height,
+                hash: h!("not_current_hash"),
+            };
+            TestCase {
+                name: "inconsistent_current_block",
+                chain: local_chain![(0, h!("_")), (height, h!("not_current_hash"))],
+                header,
+                height,
+                connected_to,
+                exp_result: Err(ApplyHeaderError::InconsistentBlocks),
+            }
+        },
+        {
+            let header = header_from_prev_blockhash(h!("B"));
+            let height = 3;
+            let connected_to = BlockId {
+                height: 4,
+                hash: h!("D"),
+            };
+            TestCase {
+                name: "connected_to_is_greater",
+                chain: local_chain![(0, h!("_")), (2, h!("B"))],
+                header,
+                height,
+                connected_to,
+                exp_result: Err(ApplyHeaderError::InconsistentBlocks),
+            }
+        },
+    ];
+
+    for (i, t) in test_cases.into_iter().enumerate() {
+        println!("running test case {}: '{}'", i, t.name);
+        let mut chain = t.chain;
+        let result = chain.apply_header_connected_to(&t.header, t.height, t.connected_to);
+        let exp_result = t
+            .exp_result
+            .map(|cs| cs.iter().cloned().collect::<ChangeSet>());
+        assert_eq!(result, exp_result, "[{}:{}] unexpected result", i, t.name);
+    }
+}

crates/chain/tests/test_spk_txout_index.rs

@@ -1,10 +1,10 @@
-use bdk_chain::SpkTxOutIndex;
-use bitcoin::{hashes::hex::FromHex, OutPoint, PackedLockTime, Script, Transaction, TxIn, TxOut};
+use bdk_chain::{indexed_tx_graph::Indexer, SpkTxOutIndex};
+use bitcoin::{absolute, OutPoint, ScriptBuf, Transaction, TxIn, TxOut};
 
 #[test]
 fn spk_txout_sent_and_received() {
-    let spk1 = Script::from_hex("001404f1e52ce2bab3423c6a8c63b7cd730d8f12542c").unwrap();
-    let spk2 = Script::from_hex("00142b57404ae14f08c3a0c903feb2af7830605eb00f").unwrap();
+    let spk1 = ScriptBuf::from_hex("001404f1e52ce2bab3423c6a8c63b7cd730d8f12542c").unwrap();
+    let spk2 = ScriptBuf::from_hex("00142b57404ae14f08c3a0c903feb2af7830605eb00f").unwrap();
 
     let mut index = SpkTxOutIndex::default();
     index.insert_spk(0, spk1.clone());
@@ -12,7 +12,7 @@ fn spk_txout_sent_and_received() {
 
     let tx1 = Transaction {
         version: 0x02,
-        lock_time: PackedLockTime(0),
+        lock_time: absolute::LockTime::ZERO,
         input: vec![],
         output: vec![TxOut {
             value: 42_000,
@@ -22,7 +22,7 @@ fn spk_txout_sent_and_received() {
 
     assert_eq!(index.sent_and_received(&tx1), (0, 42_000));
     assert_eq!(index.net_value(&tx1), 42_000);
-    index.scan(&tx1);
+    index.index_tx(&tx1);
     assert_eq!(
         index.sent_and_received(&tx1),
         (0, 42_000),
@@ -31,7 +31,7 @@ fn spk_txout_sent_and_received() {
 
     let tx2 = Transaction {
         version: 0x1,
-        lock_time: PackedLockTime(0),
+        lock_time: absolute::LockTime::ZERO,
         input: vec![TxIn {
             previous_output: OutPoint {
                 txid: tx1.txid(),
@@ -57,8 +57,8 @@ fn spk_txout_sent_and_received() {
 
 #[test]
 fn mark_used() {
-    let spk1 = Script::from_hex("001404f1e52ce2bab3423c6a8c63b7cd730d8f12542c").unwrap();
-    let spk2 = Script::from_hex("00142b57404ae14f08c3a0c903feb2af7830605eb00f").unwrap();
+    let spk1 = ScriptBuf::from_hex("001404f1e52ce2bab3423c6a8c63b7cd730d8f12542c").unwrap();
+    let spk2 = ScriptBuf::from_hex("00142b57404ae14f08c3a0c903feb2af7830605eb00f").unwrap();
 
     let mut spk_index = SpkTxOutIndex::default();
     spk_index.insert_spk(1, spk1.clone());
@@ -74,7 +74,7 @@ fn mark_used() {
 
     let tx1 = Transaction {
         version: 0x02,
-        lock_time: PackedLockTime(0),
+        lock_time: absolute::LockTime::ZERO,
         input: vec![],
         output: vec![TxOut {
             value: 42_000,
@@ -82,7 +82,7 @@ fn mark_used() {
         }],
     };
 
-    spk_index.scan(&tx1);
+    spk_index.index_tx(&tx1);
     spk_index.unmark_used(&1);
     assert!(
         spk_index.is_used(&1),

crates/chain/tests/test_tx_graph_conflicts.rs

@@ -0,0 +1,668 @@
+#[macro_use]
+mod common;
+
+use std::collections::{BTreeSet, HashSet};
+
+use bdk_chain::{keychain::Balance, BlockId};
+use bitcoin::{OutPoint, Script};
+use common::*;
+
+#[allow(dead_code)]
+struct Scenario<'a> {
+    /// Name of the test scenario
+    name: &'a str,
+    /// Transaction templates
+    tx_templates: &'a [TxTemplate<'a, BlockId>],
+    /// Names of txs that must exist in the output of `list_chain_txs`
+    exp_chain_txs: HashSet<&'a str>,
+    /// Outpoints that must exist in the output of `filter_chain_txouts`
+    exp_chain_txouts: HashSet<(&'a str, u32)>,
+    /// Outpoints of UTXOs that must exist in the output of `filter_chain_unspents`
+    exp_unspents: HashSet<(&'a str, u32)>,
+    /// Expected balances
+    exp_balance: Balance,
+}
+
+/// This test ensures that [`TxGraph`] will reliably filter out irrelevant transactions when
+/// presented with multiple conflicting transaction scenarios using the [`TxTemplate`] structure.
+/// This test also checks that [`TxGraph::list_chain_txs`], [`TxGraph::filter_chain_txouts`],
+/// [`TxGraph::filter_chain_unspents`], and [`TxGraph::balance`] return correct data.
+#[test]
+fn test_tx_conflict_handling() {
+    // Create Local chains
+    let local_chain = local_chain!(
+        (0, h!("A")),
+        (1, h!("B")),
+        (2, h!("C")),
+        (3, h!("D")),
+        (4, h!("E")),
+        (5, h!("F")),
+        (6, h!("G"))
+    );
+    let chain_tip = local_chain.tip().block_id();
+
+    let scenarios = [
+        Scenario {
+            name: "coinbase tx cannot be in mempool and be unconfirmed",
+            tx_templates: &[
+                TxTemplate {
+                    tx_name: "unconfirmed_coinbase",
+                    inputs: &[TxInTemplate::Coinbase],
+                    outputs: &[TxOutTemplate::new(5000, Some(0))],
+                    ..Default::default()
+                },
+                TxTemplate {
+                    tx_name: "confirmed_genesis",
+                    inputs: &[TxInTemplate::Bogus],
+                    outputs: &[TxOutTemplate::new(10000, Some(1))],
+                    anchors: &[block_id!(1, "B")],
+                    last_seen: None,
+                },
+                TxTemplate {
+                    tx_name: "unconfirmed_conflict",
+                    inputs: &[
+                        TxInTemplate::PrevTx("confirmed_genesis", 0), 
+                        TxInTemplate::PrevTx("unconfirmed_coinbase", 0)
+                    ],
+                    outputs: &[TxOutTemplate::new(20000, Some(2))],
+                    ..Default::default()
+                },
+                TxTemplate {
+                    tx_name: "confirmed_conflict",
+                    inputs: &[TxInTemplate::PrevTx("confirmed_genesis", 0)],
+                    outputs: &[TxOutTemplate::new(20000, Some(3))],
+                    anchors: &[block_id!(4, "E")],
+                    ..Default::default()
+                },
+            ],
+            exp_chain_txs: HashSet::from(["confirmed_genesis", "confirmed_conflict"]),
+            exp_chain_txouts: HashSet::from([("confirmed_genesis", 0), ("confirmed_conflict", 0)]),
+            exp_unspents: HashSet::from([("confirmed_conflict", 0)]),
+            exp_balance: Balance {
+                immature: 0,
+                trusted_pending: 0,
+                untrusted_pending: 0,
+                confirmed: 20000,
+            },
+        },
+        Scenario {
+            name: "2 unconfirmed txs with same last_seens conflict",
+            tx_templates: &[
+                TxTemplate {
+                    tx_name: "tx1",
+                    outputs: &[TxOutTemplate::new(40000, Some(0))],
+                    anchors: &[block_id!(1, "B")],
+                    last_seen: None,
+                    ..Default::default()
+                },
+                TxTemplate {
+                    tx_name: "tx_conflict_1",
+                    inputs: &[TxInTemplate::PrevTx("tx1", 0)],
+                    outputs: &[TxOutTemplate::new(20000, Some(2))],
+                    last_seen: Some(300),
+                    ..Default::default()
+                },
+                TxTemplate {
+                    tx_name: "tx_conflict_2",
+                    inputs: &[TxInTemplate::PrevTx("tx1", 0)],
+                    outputs: &[TxOutTemplate::new(30000, Some(3))],
+                    last_seen: Some(300),
+                    ..Default::default()
+                },
+            ],
+            // the txgraph is going to pick tx_conflict_2 because of higher lexicographical txid
+            exp_chain_txs: HashSet::from(["tx1", "tx_conflict_2"]),
+            exp_chain_txouts: HashSet::from([("tx1", 0), ("tx_conflict_2", 0)]),
+            exp_unspents: HashSet::from([("tx_conflict_2", 0)]),
+            exp_balance: Balance {
+                immature: 0,
+                trusted_pending: 30000,
+                untrusted_pending: 0,
+                confirmed: 0,
+            },
+        },
+        Scenario {
+            name: "2 unconfirmed txs with different last_seens conflict",
+            tx_templates: &[
+                TxTemplate {
+                    tx_name: "tx1",
+                    inputs: &[TxInTemplate::Bogus],
+                    outputs: &[TxOutTemplate::new(10000, Some(0)), TxOutTemplate::new(10000, Some(1))],
+                    anchors: &[block_id!(1, "B")],
+                    last_seen: None,
+                },
+                TxTemplate {
+                    tx_name: "tx_conflict_1",
+                    inputs: &[TxInTemplate::PrevTx("tx1", 0), TxInTemplate::Bogus],
+                    outputs: &[TxOutTemplate::new(20000, Some(2))],
+                    last_seen: Some(200),
+                    ..Default::default()
+                },
+                TxTemplate {
+                    tx_name: "tx_conflict_2",
+                    inputs: &[TxInTemplate::PrevTx("tx1", 0),  TxInTemplate::PrevTx("tx1", 1)],
+                    outputs: &[TxOutTemplate::new(30000, Some(3))],
+                    last_seen: Some(300),
+                    ..Default::default()
+                },
+            ],
+            exp_chain_txs: HashSet::from(["tx1", "tx_conflict_2"]),
+            exp_chain_txouts: HashSet::from([("tx1", 0), ("tx1", 1), ("tx_conflict_2", 0)]),
+            exp_unspents: HashSet::from([("tx_conflict_2", 0)]),
+            exp_balance: Balance {
+                immature: 0,
+                trusted_pending: 30000,
+                untrusted_pending: 0,
+                confirmed: 0,
+            },
+        },
+        Scenario {
+            name: "3 unconfirmed txs with different last_seens conflict",
+            tx_templates: &[
+                TxTemplate {
+                    tx_name: "tx1",
+                    inputs: &[TxInTemplate::Bogus],
+                    outputs: &[TxOutTemplate::new(10000, Some(0))],
+                    anchors: &[block_id!(1, "B")],
+                    last_seen: None,
+                },
+                TxTemplate {
+                    tx_name: "tx_conflict_1",
+                    inputs: &[TxInTemplate::PrevTx("tx1", 0), TxInTemplate::Bogus],
+                    outputs: &[TxOutTemplate::new(20000, Some(1))],
+                    last_seen: Some(200),
+                    ..Default::default()
+                },
+                TxTemplate {
+                    tx_name: "tx_conflict_2",
+                    inputs: &[TxInTemplate::PrevTx("tx1", 0)],
+                    outputs: &[TxOutTemplate::new(30000, Some(2))],
+                    last_seen: Some(300),
+                    ..Default::default()
+                },
+                TxTemplate {
+                    tx_name: "tx_conflict_3",
+                    inputs: &[TxInTemplate::PrevTx("tx1", 0)],
+                    outputs: &[TxOutTemplate::new(40000, Some(3))],
+                    last_seen: Some(400),
+                    ..Default::default()
+                },
+            ],
+            exp_chain_txs: HashSet::from(["tx1", "tx_conflict_3"]),
+            exp_chain_txouts: HashSet::from([("tx1", 0), ("tx_conflict_3", 0)]),
+            exp_unspents: HashSet::from([("tx_conflict_3", 0)]),
+            exp_balance: Balance {
+                immature: 0,
+                trusted_pending: 40000,
+                untrusted_pending: 0,
+                confirmed: 0,
+            },
+        },
+        Scenario {
+            name: "unconfirmed tx conflicts with tx in orphaned block, orphaned higher last_seen",
+            tx_templates: &[
+                TxTemplate {
+                    tx_name: "tx1",
+                    inputs: &[TxInTemplate::Bogus],
+                    outputs: &[TxOutTemplate::new(10000, Some(0))],
+                    anchors: &[block_id!(1, "B")],
+                    last_seen: None,
+                },
+                TxTemplate {
+                    tx_name: "tx_conflict_1",
+                    inputs: &[TxInTemplate::PrevTx("tx1", 0), TxInTemplate::Bogus],
+                    outputs: &[TxOutTemplate::new(20000, Some(1))],
+                    last_seen: Some(200),
+                    ..Default::default()
+                },
+                TxTemplate {
+                    tx_name: "tx_orphaned_conflict",
+                    inputs: &[TxInTemplate::PrevTx("tx1", 0)],
+                    outputs: &[TxOutTemplate::new(30000, Some(2))],
+                    anchors: &[block_id!(4, "Orphaned Block")],
+                    last_seen: Some(300),
+                },
+            ],
+            exp_chain_txs: HashSet::from(["tx1", "tx_orphaned_conflict"]),
+            exp_chain_txouts: HashSet::from([("tx1", 0), ("tx_orphaned_conflict", 0)]),
+            exp_unspents: HashSet::from([("tx_orphaned_conflict", 0)]),
+            exp_balance: Balance {
+                immature: 0,
+                trusted_pending: 30000,
+                untrusted_pending: 0,
+                confirmed: 0,
+            },
+        },
+        Scenario {
+            name: "unconfirmed tx conflicts with tx in orphaned block, orphaned lower last_seen",
+            tx_templates: &[
+                TxTemplate {
+                    tx_name: "tx1",
+                    inputs: &[TxInTemplate::Bogus],
+                    outputs: &[TxOutTemplate::new(10000, Some(0))],
+                    anchors: &[block_id!(1, "B")],
+                    last_seen: None,
+                },
+                TxTemplate {
+                    tx_name: "tx_conflict_1",
+                    inputs: &[TxInTemplate::PrevTx("tx1", 0), TxInTemplate::Bogus],
+                    outputs: &[TxOutTemplate::new(20000, Some(1))],
+                    last_seen: Some(200),
+                    ..Default::default()
+                },
+                TxTemplate {
+                    tx_name: "tx_orphaned_conflict",
+                    inputs: &[TxInTemplate::PrevTx("tx1", 0)],
+                    outputs: &[TxOutTemplate::new(30000, Some(2))],
+                    anchors: &[block_id!(4, "Orphaned Block")],
+                    last_seen: Some(100),
+                },
+            ],
+            exp_chain_txs: HashSet::from(["tx1", "tx_conflict_1"]),
+            exp_chain_txouts: HashSet::from([("tx1", 0), ("tx_conflict_1", 0)]),
+            exp_unspents: HashSet::from([("tx_conflict_1", 0)]),
+            exp_balance: Balance {
+                immature: 0,
+                trusted_pending: 20000,
+                untrusted_pending: 0,
+                confirmed: 0,
+            },
+        },
+        Scenario {
+            name: "multiple unconfirmed txs conflict with a confirmed tx",
+            tx_templates: &[
+                TxTemplate {
+                    tx_name: "tx1",
+                    inputs: &[TxInTemplate::Bogus],
+                    outputs: &[TxOutTemplate::new(10000, Some(0))],
+                    anchors: &[block_id!(1, "B")],
+                    last_seen: None,
+                },
+                TxTemplate {
+                    tx_name: "tx_conflict_1",
+                    inputs: &[TxInTemplate::PrevTx("tx1", 0), TxInTemplate::Bogus],
+                    outputs: &[TxOutTemplate::new(20000, Some(1))],
+                    last_seen: Some(200),
+                    ..Default::default()
+                },
+                TxTemplate {
+                    tx_name: "tx_conflict_2",
+                    inputs: &[TxInTemplate::PrevTx("tx1", 0)],
+                    outputs: &[TxOutTemplate::new(30000, Some(2))],
+                    last_seen: Some(300),
+                    ..Default::default()
+                },
+                TxTemplate {
+                    tx_name: "tx_conflict_3",
+                    inputs: &[TxInTemplate::PrevTx("tx1", 0)],
+                    outputs: &[TxOutTemplate::new(40000, Some(3))],
+                    last_seen: Some(400),
+                    ..Default::default()
+                },
+                TxTemplate {
+                    tx_name: "tx_confirmed_conflict",
+                    inputs: &[TxInTemplate::PrevTx("tx1", 0)],
+                    outputs: &[TxOutTemplate::new(50000, Some(4))],
+                    anchors: &[block_id!(1, "B")],
+                    ..Default::default()
+                },
+            ],
+            exp_chain_txs: HashSet::from(["tx1", "tx_confirmed_conflict"]),
+            exp_chain_txouts: HashSet::from([("tx1", 0), ("tx_confirmed_conflict", 0)]),
+            exp_unspents: HashSet::from([("tx_confirmed_conflict", 0)]),
+            exp_balance: Balance {
+                immature: 0,
+                trusted_pending: 0,
+                untrusted_pending: 0,
+                confirmed: 50000,
+            },
+        },
+        Scenario {
+            name: "B and B' spend A and conflict, C spends B, all the transactions are unconfirmed, B' has higher last_seen than B",
+            tx_templates: &[
+                TxTemplate {
+                    tx_name: "A",
+                    inputs: &[TxInTemplate::Bogus],
+                    outputs: &[TxOutTemplate::new(10000, Some(0))],
+                    last_seen: Some(22),
+                    ..Default::default()
+                },
+                TxTemplate {
+                    tx_name: "B",
+                    inputs: &[TxInTemplate::PrevTx("A", 0)],
+                    outputs: &[TxOutTemplate::new(20000, Some(1))],
+                    last_seen: Some(23),
+                    ..Default::default()
+                },
+                TxTemplate {
+                    tx_name: "B'",
+                    inputs: &[TxInTemplate::PrevTx("A", 0)],
+                    outputs: &[TxOutTemplate::new(20000, Some(2))],
+                    last_seen: Some(24),
+                    ..Default::default()
+                },
+                TxTemplate {
+                    tx_name: "C",
+                    inputs: &[TxInTemplate::PrevTx("B", 0)],
+                    outputs: &[TxOutTemplate::new(30000, Some(3))],
+                    last_seen: Some(25),
+                    ..Default::default()
+                },
+            ],
+            // A, B, C will appear in the list methods
+            // This is because B' has a higher last seen than B, but C has a higher
+            // last seen than B', so B and C are considered canonical
+            exp_chain_txs: HashSet::from(["A", "B", "C"]),
+            exp_chain_txouts: HashSet::from([("A", 0), ("B", 0), ("C", 0)]),
+            exp_unspents: HashSet::from([("C", 0)]),
+            exp_balance: Balance {
+                immature: 0,
+                trusted_pending: 30000,
+                untrusted_pending: 0,
+                confirmed: 0,
+            },
+        },
+        Scenario {
+            name: "B and B' spend A and conflict, C spends B, A and B' are in best chain",
+            tx_templates: &[
+                TxTemplate {
+                    tx_name: "A",
+                    inputs: &[TxInTemplate::Bogus],
+                    outputs: &[TxOutTemplate::new(10000, Some(0))],
+                    anchors: &[block_id!(1, "B")],
+                    last_seen: None,
+                },
+                TxTemplate {
+                    tx_name: "B",
+                    inputs: &[TxInTemplate::PrevTx("A", 0)],
+                    outputs: &[TxOutTemplate::new(20000, Some(1))],
+                    ..Default::default()
+                },
+                TxTemplate {
+                    tx_name: "B'",
+                    inputs: &[TxInTemplate::PrevTx("A", 0)],
+                    outputs: &[TxOutTemplate::new(20000, Some(2))],
+                    anchors: &[block_id!(4, "E")],
+                    ..Default::default()
+                },
+                TxTemplate {
+                    tx_name: "C",
+                    inputs: &[TxInTemplate::PrevTx("B", 0)],
+                    outputs: &[TxOutTemplate::new(30000, Some(3))],
+                    ..Default::default()
+                },
+            ],
+            // B and C should not appear in the list methods
+            exp_chain_txs: HashSet::from(["A", "B'"]),
+            exp_chain_txouts: HashSet::from([("A", 0), ("B'", 0)]),
+            exp_unspents: HashSet::from([("B'", 0)]),
+            exp_balance: Balance {
+                immature: 0,
+                trusted_pending: 0,
+                untrusted_pending: 0,
+                confirmed: 20000,
+            },
+        },
+        Scenario {
+            name: "B and B' spend A and conflict, C spends B', A and B' are in best chain",
+            tx_templates: &[
+                TxTemplate {
+                    tx_name: "A",
+                    inputs: &[TxInTemplate::Bogus],
+                    outputs: &[TxOutTemplate::new(10000, Some(0))],
+                    anchors: &[block_id!(1, "B")],
+                    last_seen: None,
+                },
+                TxTemplate {
+                    tx_name: "B",
+                    inputs: &[TxInTemplate::PrevTx("A", 0)],
+                    outputs: &[TxOutTemplate::new(20000, Some(1))],
+                    ..Default::default()
+                },
+                TxTemplate {
+                    tx_name: "B'",
+                    inputs: &[TxInTemplate::PrevTx("A", 0)],
+                    outputs: &[TxOutTemplate::new(20000, Some(2))],
+                    anchors: &[block_id!(4, "E")],
+                    ..Default::default()
+                },
+                TxTemplate {
+                    tx_name: "C",
+                    inputs: &[TxInTemplate::PrevTx("B'", 0)],
+                    outputs: &[TxOutTemplate::new(30000, Some(3))],
+                    ..Default::default()
+                },
+            ],
+            // B should not appear in the list methods
+            exp_chain_txs: HashSet::from(["A", "B'", "C"]),
+            exp_chain_txouts: HashSet::from([
+                ("A", 0),
+                ("B'", 0),
+                ("C", 0),
+            ]),
+            exp_unspents: HashSet::from([("C", 0)]),
+            exp_balance: Balance {
+                immature: 0,
+                trusted_pending: 30000,
+                untrusted_pending: 0,
+                confirmed: 0,
+            },
+        },
+        Scenario {
+            name: "B and B' spend A and conflict, C spends both B and B', A is in best chain",
+            tx_templates: &[
+                TxTemplate {
+                    tx_name: "A",
+                    inputs: &[TxInTemplate::Bogus],
+                    outputs: &[TxOutTemplate::new(10000, Some(0))],
+                    anchors: &[block_id!(1, "B")],
+                    last_seen: None,
+                },
+                TxTemplate {
+                    tx_name: "B",
+                    inputs: &[TxInTemplate::PrevTx("A", 0), TxInTemplate::Bogus],
+                    outputs: &[TxOutTemplate::new(20000, Some(1))],
+                    last_seen: Some(200),
+                    ..Default::default()
+                },
+                TxTemplate {
+                    tx_name: "B'",
+                    inputs: &[TxInTemplate::PrevTx("A", 0)],
+                    outputs: &[TxOutTemplate::new(30000, Some(2))],
+                    last_seen: Some(300),
+                    ..Default::default()
+                },
+                TxTemplate {
+                    tx_name: "C",
+                    inputs: &[
+                        TxInTemplate::PrevTx("B", 0),
+                        TxInTemplate::PrevTx("B'", 0),
+                    ],
+                    outputs: &[TxOutTemplate::new(20000, Some(3))],
+                    ..Default::default()
+                },
+            ],
+            // C should not appear in the list methods
+            exp_chain_txs: HashSet::from(["A", "B'"]),
+            exp_chain_txouts: HashSet::from([("A", 0), ("B'", 0)]),
+            exp_unspents: HashSet::from([("B'", 0)]),
+            exp_balance: Balance {
+                immature: 0,
+                trusted_pending: 30000,
+                untrusted_pending: 0,
+                confirmed: 0,
+            },
+        },
+        Scenario {
+            name: "B and B' spend A and conflict, B' is confirmed, C spends both B and B', A is in best chain",
+            tx_templates: &[
+                TxTemplate {
+                    tx_name: "A",
+                    inputs: &[TxInTemplate::Bogus],
+                    outputs: &[TxOutTemplate::new(10000, Some(0))],
+                    anchors: &[block_id!(1, "B")],
+                    last_seen: None,
+                },
+                TxTemplate {
+                    tx_name: "B",
+                    inputs: &[TxInTemplate::PrevTx("A", 0), TxInTemplate::Bogus],
+                    outputs: &[TxOutTemplate::new(20000, Some(1))],
+                    last_seen: Some(200),
+                    ..Default::default()
+                },
+                TxTemplate {
+                    tx_name: "B'",
+                    inputs: &[TxInTemplate::PrevTx("A", 0)],
+                    outputs: &[TxOutTemplate::new(50000, Some(4))],
+                    anchors: &[block_id!(1, "B")],
+                    ..Default::default()
+                },
+                TxTemplate {
+                    tx_name: "C",
+                    inputs: &[
+                        TxInTemplate::PrevTx("B", 0),
+                        TxInTemplate::PrevTx("B'", 0),
+                    ],
+                    outputs: &[TxOutTemplate::new(20000, Some(5))],
+                    ..Default::default()
+                },
+            ],
+            // C should not appear in the list methods
+            exp_chain_txs: HashSet::from(["A", "B'"]),
+            exp_chain_txouts: HashSet::from([("A", 0), ("B'", 0)]),
+            exp_unspents: HashSet::from([("B'", 0)]),
+            exp_balance: Balance {
+                immature: 0,
+                trusted_pending: 0,
+                untrusted_pending: 0,
+                confirmed: 50000,
+            },
+        },
+        Scenario {
+            name: "B and B' spend A and conflict, B' is confirmed, C spends both B and B', D spends C, A is in best chain",
+            tx_templates: &[
+                TxTemplate {
+                    tx_name: "A",
+                    inputs: &[TxInTemplate::Bogus],
+                    outputs: &[TxOutTemplate::new(10000, Some(0))],
+                    anchors: &[block_id!(1, "B")],
+                    last_seen: None,
+                },
+                TxTemplate {
+                    tx_name: "B",
+                    inputs: &[TxInTemplate::PrevTx("A", 0), TxInTemplate::Bogus],
+                    outputs: &[TxOutTemplate::new(20000, Some(1))],
+                    last_seen: Some(200),
+                    ..Default::default()
+                },
+                TxTemplate {
+                    tx_name: "B'",
+                    inputs: &[TxInTemplate::PrevTx("A", 0)],
+                    outputs: &[TxOutTemplate::new(50000, Some(4))],
+                    anchors: &[block_id!(1, "B")],
+                    ..Default::default()
+                },
+                TxTemplate {
+                    tx_name: "C",
+                    inputs: &[
+                        TxInTemplate::PrevTx("B", 0),
+                        TxInTemplate::PrevTx("B'", 0),
+                    ],
+                    outputs: &[TxOutTemplate::new(20000, Some(5))],
+                    ..Default::default()
+                },
+                TxTemplate {
+                    tx_name: "D",
+                    inputs: &[TxInTemplate::PrevTx("C", 0)],
+                    outputs: &[TxOutTemplate::new(20000, Some(6))],
+                    ..Default::default()
+                },
+            ],
+            // D should not appear in the list methods
+            exp_chain_txs: HashSet::from(["A", "B'"]),
+            exp_chain_txouts: HashSet::from([("A", 0), ("B'", 0)]),
+            exp_unspents: HashSet::from([("B'", 0)]),
+            exp_balance: Balance {
+                immature: 0,
+                trusted_pending: 0,
+                untrusted_pending: 0,
+                confirmed: 50000,
+            },
+        },
+    ];
+
+    for scenario in scenarios {
+        let (tx_graph, spk_index, exp_tx_ids) = init_graph(scenario.tx_templates.iter());
+
+        let txs = tx_graph
+            .list_chain_txs(&local_chain, chain_tip)
+            .map(|tx| tx.tx_node.txid)
+            .collect::<BTreeSet<_>>();
+        let exp_txs = scenario
+            .exp_chain_txs
+            .iter()
+            .map(|txid| *exp_tx_ids.get(txid).expect("txid must exist"))
+            .collect::<BTreeSet<_>>();
+        assert_eq!(
+            txs, exp_txs,
+            "\n[{}] 'list_chain_txs' failed",
+            scenario.name
+        );
+
+        let txouts = tx_graph
+            .filter_chain_txouts(
+                &local_chain,
+                chain_tip,
+                spk_index.outpoints().iter().cloned(),
+            )
+            .map(|(_, full_txout)| full_txout.outpoint)
+            .collect::<BTreeSet<_>>();
+        let exp_txouts = scenario
+            .exp_chain_txouts
+            .iter()
+            .map(|(txid, vout)| OutPoint {
+                txid: *exp_tx_ids.get(txid).expect("txid must exist"),
+                vout: *vout,
+            })
+            .collect::<BTreeSet<_>>();
+        assert_eq!(
+            txouts, exp_txouts,
+            "\n[{}] 'filter_chain_txouts' failed",
+            scenario.name
+        );
+
+        let utxos = tx_graph
+            .filter_chain_unspents(
+                &local_chain,
+                chain_tip,
+                spk_index.outpoints().iter().cloned(),
+            )
+            .map(|(_, full_txout)| full_txout.outpoint)
+            .collect::<BTreeSet<_>>();
+        let exp_utxos = scenario
+            .exp_unspents
+            .iter()
+            .map(|(txid, vout)| OutPoint {
+                txid: *exp_tx_ids.get(txid).expect("txid must exist"),
+                vout: *vout,
+            })
+            .collect::<BTreeSet<_>>();
+        assert_eq!(
+            utxos, exp_utxos,
+            "\n[{}] 'filter_chain_unspents' failed",
+            scenario.name
+        );
+
+        let balance = tx_graph.balance(
+            &local_chain,
+            chain_tip,
+            spk_index.outpoints().iter().cloned(),
+            |_, spk: &Script| spk_index.index_of_spk(spk).is_some(),
+        );
+        assert_eq!(
+            balance, scenario.exp_balance,
+            "\n[{}] 'balance' failed",
+            scenario.name
+        );
+    }
+}

crates/electrum/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "bdk_electrum"
-version = "0.3.0"
+version = "0.9.0"
 edition = "2021"
 homepage = "https://bitcoindevkit.org"
 repository = "https://github.com/bitcoindevkit/bdk"
@@ -12,5 +12,6 @@ readme = "README.md"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-bdk_chain = { path = "../chain", version = "0.5.0", features = ["serde", "miniscript"] }
-electrum-client = { version = "0.12" }
+bdk_chain = { path = "../chain", version = "0.11.0", default-features = false }
+electrum-client = { version = "0.18" }
+#rustls = { version = "=0.21.1", optional = true, features = ["dangerous_configuration"] }

crates/electrum/README.md

@@ -1,3 +1,7 @@
 # BDK Electrum
 
-BDK Electrum client library for updating the keychain tracker.
+BDK Electrum extends [`electrum-client`] to update [`bdk_chain`] structures
+from an Electrum server.
+
+[`electrum-client`]: https://docs.rs/electrum-client/
+[`bdk_chain`]: https://docs.rs/bdk-chain/

crates/electrum/src/electrum_ext.rs

@@ -1,85 +1,80 @@
 use bdk_chain::{
-    bitcoin::{hashes::hex::FromHex, BlockHash, OutPoint, Script, Transaction, Txid},
-    keychain::LocalUpdate,
-    local_chain::LocalChain,
+    bitcoin::{OutPoint, ScriptBuf, Transaction, Txid},
+    local_chain::{self, CheckPoint},
     tx_graph::{self, TxGraph},
-    Anchor, BlockId, ConfirmationHeightAnchor, ConfirmationTimeAnchor,
+    Anchor, BlockId, ConfirmationHeightAnchor, ConfirmationTimeHeightAnchor,
 };
-use electrum_client::{Client, ElectrumApi, Error};
+use electrum_client::{Client, ElectrumApi, Error, HeaderNotification};
 use std::{
     collections::{BTreeMap, BTreeSet, HashMap, HashSet},
     fmt::Debug,
+    str::FromStr,
 };
 
-#[derive(Debug, Clone)]
-pub struct ElectrumUpdate<K, A> {
-    pub graph_update: HashMap<Txid, BTreeSet<A>>,
-    pub chain_update: LocalChain,
-    pub keychain_update: BTreeMap<K, u32>,
-}
+/// We include a chain suffix of a certain length for the purpose of robustness.
+const CHAIN_SUFFIX_LENGTH: u32 = 8;
 
-impl<K, A> Default for ElectrumUpdate<K, A> {
-    fn default() -> Self {
-        Self {
-            graph_update: Default::default(),
-            chain_update: Default::default(),
-            keychain_update: Default::default(),
-        }
-    }
-}
+/// Represents updates fetched from an Electrum server, but excludes full transactions.
+///
+/// To provide a complete update to [`TxGraph`], you'll need to call [`Self::missing_full_txs`] to
+/// determine the full transactions missing from [`TxGraph`]. Then call [`Self::into_tx_graph`] to
+/// fetch the full transactions from Electrum and finalize the update.
+#[derive(Debug, Default, Clone)]
+pub struct RelevantTxids(HashMap<Txid, BTreeSet<ConfirmationHeightAnchor>>);
 
-impl<K, A: Anchor> ElectrumUpdate<K, A> {
-    pub fn missing_full_txs<A2>(&self, graph: &TxGraph<A2>) -> Vec<Txid> {
-        self.graph_update
+impl RelevantTxids {
+    /// Determine the full transactions that are missing from `graph`.
+    ///
+    /// Refer to [`RelevantTxids`] for more details.
+    pub fn missing_full_txs<A: Anchor>(&self, graph: &TxGraph<A>) -> Vec<Txid> {
+        self.0
             .keys()
             .filter(move |&&txid| graph.as_ref().get_tx(txid).is_none())
             .cloned()
             .collect()
     }
 
-    pub fn finalize(
+    /// Finalizes the [`TxGraph`] update by fetching `missing` txids from the `client`.
+    ///
+    /// Refer to [`RelevantTxids`] for more details.
+    pub fn into_tx_graph(
         self,
         client: &Client,
         seen_at: Option<u64>,
         missing: Vec<Txid>,
-    ) -> Result<LocalUpdate<K, A>, Error> {
+    ) -> Result<TxGraph<ConfirmationHeightAnchor>, Error> {
         let new_txs = client.batch_transaction_get(&missing)?;
-        let mut graph_update = TxGraph::<A>::new(new_txs);
-        for (txid, anchors) in self.graph_update {
+        let mut graph = TxGraph::<ConfirmationHeightAnchor>::new(new_txs);
+        for (txid, anchors) in self.0 {
             if let Some(seen_at) = seen_at {
-                let _ = graph_update.insert_seen_at(txid, seen_at);
+                let _ = graph.insert_seen_at(txid, seen_at);
             }
             for anchor in anchors {
-                let _ = graph_update.insert_anchor(txid, anchor);
+                let _ = graph.insert_anchor(txid, anchor);
             }
         }
-        Ok(LocalUpdate {
-            keychain: self.keychain_update,
-            graph: graph_update,
-            chain: self.chain_update,
-        })
+        Ok(graph)
     }
-}
 
-impl<K> ElectrumUpdate<K, ConfirmationHeightAnchor> {
-    /// Finalizes the [`ElectrumUpdate`] with `new_txs` and anchors of type
-    /// [`ConfirmationTimeAnchor`].
+    /// Finalizes the update by fetching `missing` txids from the `client`, where the
+    /// resulting [`TxGraph`] has anchors of type [`ConfirmationTimeHeightAnchor`].
+    ///
+    /// Refer to [`RelevantTxids`] for more details.
     ///
     /// **Note:** The confirmation time might not be precisely correct if there has been a reorg.
-    /// Electrum's API intends that we use the merkle proof API, we should change `bdk_electrum` to
-    /// use it.
-    pub fn finalize_as_confirmation_time(
+    // Electrum's API intends that we use the merkle proof API, we should change `bdk_electrum` to
+    // use it.
+    pub fn into_confirmation_time_tx_graph(
         self,
         client: &Client,
         seen_at: Option<u64>,
         missing: Vec<Txid>,
-    ) -> Result<LocalUpdate<K, ConfirmationTimeAnchor>, Error> {
-        let update = self.finalize(client, seen_at, missing)?;
+    ) -> Result<TxGraph<ConfirmationTimeHeightAnchor>, Error> {
+        let graph = self.into_tx_graph(client, seen_at, missing)?;
 
         let relevant_heights = {
             let mut visited_heights = HashSet::new();
-            update
-                .graph
+            graph
                 .all_anchors()
                 .iter()
                 .map(|(a, _)| a.confirmation_height_upper_bound())
@@ -98,19 +93,19 @@ impl<K> ElectrumUpdate<K, ConfirmationHeightAnchor> {
             )
             .collect::<HashMap<u32, u64>>();
 
-        let graph_additions = {
-            let old_additions = TxGraph::default().determine_additions(&update.graph);
-            tx_graph::Additions {
-                txs: old_additions.txs,
-                txouts: old_additions.txouts,
-                last_seen: old_additions.last_seen,
-                anchors: old_additions
+        let graph_changeset = {
+            let old_changeset = TxGraph::default().apply_update(graph);
+            tx_graph::ChangeSet {
+                txs: old_changeset.txs,
+                txouts: old_changeset.txouts,
+                last_seen: old_changeset.last_seen,
+                anchors: old_changeset
                     .anchors
                     .into_iter()
                     .map(|(height_anchor, txid)| {
                         let confirmation_height = height_anchor.confirmation_height;
                         let confirmation_time = height_to_time[&confirmation_height];
-                        let time_anchor = ConfirmationTimeAnchor {
+                        let time_anchor = ConfirmationTimeHeightAnchor {
                             anchor_block: height_anchor.anchor_block,
                             confirmation_height,
                             confirmation_time,
@@ -121,96 +116,98 @@ impl<K> ElectrumUpdate<K, ConfirmationHeightAnchor> {
             }
         };
 
-        Ok(LocalUpdate {
-            keychain: update.keychain,
-            graph: {
-                let mut graph = TxGraph::default();
-                graph.apply_additions(graph_additions);
-                graph
-            },
-            chain: update.chain,
-        })
+        let mut new_graph = TxGraph::default();
+        new_graph.apply_changeset(graph_changeset);
+        Ok(new_graph)
     }
 }
 
-pub trait ElectrumExt<A> {
-    fn get_tip(&self) -> Result<(u32, BlockHash), Error>;
-
-    fn scan<K: Ord + Clone>(
-        &self,
-        local_chain: &BTreeMap<u32, BlockHash>,
-        keychain_spks: BTreeMap<K, impl IntoIterator<Item = (u32, Script)>>,
-        txids: impl IntoIterator<Item = Txid>,
-        outpoints: impl IntoIterator<Item = OutPoint>,
-        stop_gap: usize,
-        batch_size: usize,
-    ) -> Result<ElectrumUpdate<K, A>, Error>;
-
-    fn scan_without_keychain(
-        &self,
-        local_chain: &BTreeMap<u32, BlockHash>,
-        misc_spks: impl IntoIterator<Item = Script>,
-        txids: impl IntoIterator<Item = Txid>,
-        outpoints: impl IntoIterator<Item = OutPoint>,
-        batch_size: usize,
-    ) -> Result<ElectrumUpdate<(), A>, Error> {
-        let spk_iter = misc_spks
-            .into_iter()
-            .enumerate()
-            .map(|(i, spk)| (i as u32, spk));
-
-        self.scan(
-            local_chain,
-            [((), spk_iter)].into(),
-            txids,
-            outpoints,
-            usize::MAX,
-            batch_size,
-        )
-    }
+/// Combination of chain and transactions updates from electrum
+///
+/// We have to update the chain and the txids at the same time since we anchor the txids to
+/// the same chain tip that we check before and after we gather the txids.
+#[derive(Debug)]
+pub struct ElectrumUpdate {
+    /// Chain update
+    pub chain_update: local_chain::Update,
+    /// Transaction updates from electrum
+    pub relevant_txids: RelevantTxids,
 }
 
-impl ElectrumExt<ConfirmationHeightAnchor> for Client {
-    fn get_tip(&self) -> Result<(u32, BlockHash), Error> {
-        // TODO: unsubscribe when added to the client, or is there a better call to use here?
-        self.block_headers_subscribe()
-            .map(|data| (data.height as u32, data.header.block_hash()))
-    }
-
-    fn scan<K: Ord + Clone>(
+/// Trait to extend [`Client`] functionality.
+pub trait ElectrumExt {
+    /// Full scan the keychain scripts specified with the blockchain (via an Electrum client) and
+    /// returns updates for [`bdk_chain`] data structures.
+    ///
+    /// - `prev_tip`: the most recent blockchain tip present locally
+    /// - `keychain_spks`: keychains that we want to scan transactions for
+    ///
+    /// The full scan for each keychain stops after a gap of `stop_gap` script pubkeys with no associated
+    /// transactions. `batch_size` specifies the max number of script pubkeys to request for in a
+    /// single batch request.
+    fn full_scan<K: Ord + Clone>(
         &self,
-        local_chain: &BTreeMap<u32, BlockHash>,
-        keychain_spks: BTreeMap<K, impl IntoIterator<Item = (u32, Script)>>,
-        txids: impl IntoIterator<Item = Txid>,
-        outpoints: impl IntoIterator<Item = OutPoint>,
+        prev_tip: CheckPoint,
+        keychain_spks: BTreeMap<K, impl IntoIterator<Item = (u32, ScriptBuf)>>,
         stop_gap: usize,
         batch_size: usize,
-    ) -> Result<ElectrumUpdate<K, ConfirmationHeightAnchor>, Error> {
+    ) -> Result<(ElectrumUpdate, BTreeMap<K, u32>), Error>;
+
+    /// Sync a set of scripts with the blockchain (via an Electrum client) for the data specified
+    /// and returns updates for [`bdk_chain`] data structures.
+    ///
+    /// - `prev_tip`: the most recent blockchain tip present locally
+    /// - `misc_spks`: an iterator of scripts we want to sync transactions for
+    /// - `txids`: transactions for which we want updated [`Anchor`]s
+    /// - `outpoints`: transactions associated with these outpoints (residing, spending) that we
+    ///     want to include in the update
+    ///
+    /// `batch_size` specifies the max number of script pubkeys to request for in a single batch
+    /// request.
+    ///
+    /// If the scripts to sync are unknown, such as when restoring or importing a keychain that
+    /// may include scripts that have been used, use [`full_scan`] with the keychain.
+    ///
+    /// [`full_scan`]: ElectrumExt::full_scan
+    fn sync(
+        &self,
+        prev_tip: CheckPoint,
+        misc_spks: impl IntoIterator<Item = ScriptBuf>,
+        txids: impl IntoIterator<Item = Txid>,
+        outpoints: impl IntoIterator<Item = OutPoint>,
+        batch_size: usize,
+    ) -> Result<ElectrumUpdate, Error>;
+}
+
+impl<A: ElectrumApi> ElectrumExt for A {
+    fn full_scan<K: Ord + Clone>(
+        &self,
+        prev_tip: CheckPoint,
+        keychain_spks: BTreeMap<K, impl IntoIterator<Item = (u32, ScriptBuf)>>,
+        stop_gap: usize,
+        batch_size: usize,
+    ) -> Result<(ElectrumUpdate, BTreeMap<K, u32>), Error> {
         let mut request_spks = keychain_spks
             .into_iter()
-            .map(|(k, s)| (k, s.into_iter()))
+            .map(|(k, s)| (k.clone(), s.into_iter()))
             .collect::<BTreeMap<K, _>>();
-        let mut scanned_spks = BTreeMap::<(K, u32), (Script, bool)>::new();
+        let mut scanned_spks = BTreeMap::<(K, u32), (ScriptBuf, bool)>::new();
 
-        let txids = txids.into_iter().collect::<Vec<_>>();
-        let outpoints = outpoints.into_iter().collect::<Vec<_>>();
-
-        let update = loop {
-            let mut update = ElectrumUpdate::<K, ConfirmationHeightAnchor> {
-                chain_update: prepare_chain_update(self, local_chain)?,
-                ..Default::default()
-            };
-            let anchor_block = update
-                .chain_update
-                .tip()
-                .expect("must have atleast one block");
+        let (electrum_update, keychain_update) = loop {
+            let (tip, _) = construct_update_tip(self, prev_tip.clone())?;
+            let mut relevant_txids = RelevantTxids::default();
+            let cps = tip
+                .iter()
+                .take(10)
+                .map(|cp| (cp.height(), cp))
+                .collect::<BTreeMap<u32, CheckPoint>>();
 
             if !request_spks.is_empty() {
                 if !scanned_spks.is_empty() {
                     scanned_spks.append(&mut populate_with_spks(
                         self,
-                        anchor_block,
-                        &mut update,
+                        &cps,
+                        &mut relevant_txids,
                         &mut scanned_spks
                             .iter()
                             .map(|(i, (spk, _))| (i.clone(), spk.clone())),
@@ -222,8 +219,8 @@ impl ElectrumExt<ConfirmationHeightAnchor> for Client {
                     scanned_spks.extend(
                         populate_with_spks(
                             self,
-                            anchor_block,
-                            &mut update,
+                            &cps,
+                            &mut relevant_txids,
                             keychain_spks,
                             stop_gap,
                             batch_size,
@@ -234,24 +231,18 @@ impl ElectrumExt<ConfirmationHeightAnchor> for Client {
                 }
             }
 
-            populate_with_txids(self, anchor_block, &mut update, &mut txids.iter().cloned())?;
-
-            let _txs = populate_with_outpoints(
-                self,
-                anchor_block,
-                &mut update,
-                &mut outpoints.iter().cloned(),
-            )?;
-
             // check for reorgs during scan process
-            let server_blockhash = self
-                .block_header(anchor_block.height as usize)?
-                .block_hash();
-            if anchor_block.hash != server_blockhash {
+            let server_blockhash = self.block_header(tip.height() as usize)?.block_hash();
+            if tip.hash() != server_blockhash {
                 continue; // reorg
             }
 
-            update.keychain_update = request_spks
+            let chain_update = local_chain::Update {
+                tip,
+                introduce_older_blocks: true,
+            };
+
+            let keychain_update = request_spks
                 .into_keys()
                 .filter_map(|k| {
                     scanned_spks
@@ -261,53 +252,133 @@ impl ElectrumExt<ConfirmationHeightAnchor> for Client {
                         .map(|((_, i), _)| (k, *i))
                 })
                 .collect::<BTreeMap<_, _>>();
-            break update;
+
+            break (
+                ElectrumUpdate {
+                    chain_update,
+                    relevant_txids,
+                },
+                keychain_update,
+            );
         };
 
-        Ok(update)
+        Ok((electrum_update, keychain_update))
+    }
+
+    fn sync(
+        &self,
+        prev_tip: CheckPoint,
+        misc_spks: impl IntoIterator<Item = ScriptBuf>,
+        txids: impl IntoIterator<Item = Txid>,
+        outpoints: impl IntoIterator<Item = OutPoint>,
+        batch_size: usize,
+    ) -> Result<ElectrumUpdate, Error> {
+        let spk_iter = misc_spks
+            .into_iter()
+            .enumerate()
+            .map(|(i, spk)| (i as u32, spk));
+
+        let (mut electrum_update, _) = self.full_scan(
+            prev_tip.clone(),
+            [((), spk_iter)].into(),
+            usize::MAX,
+            batch_size,
+        )?;
+
+        let (tip, _) = construct_update_tip(self, prev_tip)?;
+        let cps = tip
+            .iter()
+            .take(10)
+            .map(|cp| (cp.height(), cp))
+            .collect::<BTreeMap<u32, CheckPoint>>();
+
+        populate_with_txids(self, &cps, &mut electrum_update.relevant_txids, txids)?;
+
+        let _txs =
+            populate_with_outpoints(self, &cps, &mut electrum_update.relevant_txids, outpoints)?;
+
+        Ok(electrum_update)
     }
 }
 
-/// Prepare an update "template" based on the checkpoints of the `local_chain`.
-fn prepare_chain_update(
-    client: &Client,
-    local_chain: &BTreeMap<u32, BlockHash>,
-) -> Result<LocalChain, Error> {
-    let mut update = LocalChain::default();
+/// Return a [`CheckPoint`] of the latest tip, that connects with `prev_tip`.
+fn construct_update_tip(
+    client: &impl ElectrumApi,
+    prev_tip: CheckPoint,
+) -> Result<(CheckPoint, Option<u32>), Error> {
+    let HeaderNotification { height, .. } = client.block_headers_subscribe()?;
+    let new_tip_height = height as u32;
 
-    // Find the local chain block that is still there so our update can connect to the local chain.
-    for (&existing_height, &existing_hash) in local_chain.iter().rev() {
-        // TODO: a batch request may be safer, as a reorg that happens when we are obtaining
-        //       `block_header`s will result in inconsistencies
-        let current_hash = client.block_header(existing_height as usize)?.block_hash();
-        let _ = update
-            .insert_block(BlockId {
-                height: existing_height,
-                hash: current_hash,
-            })
-            .expect("This never errors because we are working with a fresh chain");
-
-        if current_hash == existing_hash {
-            break;
-        }
+    // If electrum returns a tip height that is lower than our previous tip, then checkpoints do
+    // not need updating. We just return the previous tip and use that as the point of agreement.
+    if new_tip_height < prev_tip.height() {
+        return Ok((prev_tip.clone(), Some(prev_tip.height())));
     }
 
-    // Insert the new tip so new transactions will be accepted into the sparsechain.
-    let tip = {
-        let (height, hash) = crate::get_tip(client)?;
-        BlockId { height, hash }
+    // Atomically fetch the latest `CHAIN_SUFFIX_LENGTH` count of blocks from Electrum. We use this
+    // to construct our checkpoint update.
+    let mut new_blocks = {
+        let start_height = new_tip_height.saturating_sub(CHAIN_SUFFIX_LENGTH - 1);
+        let hashes = client
+            .block_headers(start_height as _, CHAIN_SUFFIX_LENGTH as _)?
+            .headers
+            .into_iter()
+            .map(|h| h.block_hash());
+        (start_height..).zip(hashes).collect::<BTreeMap<u32, _>>()
     };
-    if update.insert_block(tip).is_err() {
-        // There has been a re-org before we even begin scanning addresses.
-        // Just recursively call (this should never happen).
-        return prepare_chain_update(client, local_chain);
-    }
 
-    Ok(update)
+    // Find the "point of agreement" (if any).
+    let agreement_cp = {
+        let mut agreement_cp = Option::<CheckPoint>::None;
+        for cp in prev_tip.iter() {
+            let cp_block = cp.block_id();
+            let hash = match new_blocks.get(&cp_block.height) {
+                Some(&hash) => hash,
+                None => {
+                    assert!(
+                        new_tip_height >= cp_block.height,
+                        "already checked that electrum's tip cannot be smaller"
+                    );
+                    let hash = client.block_header(cp_block.height as _)?.block_hash();
+                    new_blocks.insert(cp_block.height, hash);
+                    hash
+                }
+            };
+            if hash == cp_block.hash {
+                agreement_cp = Some(cp);
+                break;
+            }
+        }
+        agreement_cp
+    };
+
+    let agreement_height = agreement_cp.as_ref().map(CheckPoint::height);
+
+    let new_tip = new_blocks
+        .into_iter()
+        // Prune `new_blocks` to only include blocks that are actually new.
+        .filter(|(height, _)| Some(*height) > agreement_height)
+        .map(|(height, hash)| BlockId { height, hash })
+        .fold(agreement_cp, |prev_cp, block| {
+            Some(match prev_cp {
+                Some(cp) => cp.push(block).expect("must extend checkpoint"),
+                None => CheckPoint::new(block),
+            })
+        })
+        .expect("must have at least one checkpoint");
+
+    Ok((new_tip, agreement_height))
 }
 
+/// A [tx status] comprises of a concatenation of `tx_hash:height:`s. We transform a single one of
+/// these concatenations into a [`ConfirmationHeightAnchor`] if possible.
+///
+/// We use the lowest possible checkpoint as the anchor block (from `cps`). If an anchor block
+/// cannot be found, or the transaction is unconfirmed, [`None`] is returned.
+///
+/// [tx status](https://electrumx-spesmilo.readthedocs.io/en/latest/protocol-basics.html#status)
 fn determine_tx_anchor(
-    anchor_block: BlockId,
+    cps: &BTreeMap<u32, CheckPoint>,
     raw_height: i32,
     txid: Txid,
 ) -> Option<ConfirmationHeightAnchor> {
@@ -316,9 +387,10 @@ fn determine_tx_anchor(
     // transactions residing in the genesis block to have height 0, then interpret a height of 0 as
     // unconfirmed for all other transactions.
     if txid
-        == Txid::from_hex("4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b")
+        == Txid::from_str("4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b")
             .expect("must deserialize genesis coinbase txid")
     {
+        let anchor_block = cps.values().next()?.block_id();
         return Some(ConfirmationHeightAnchor {
             anchor_block,
             confirmation_height: 0,
@@ -331,6 +403,7 @@ fn determine_tx_anchor(
         }
         h => {
             let h = h as u32;
+            let anchor_block = cps.range(h..).next().map(|(_, cp)| cp.block_id())?;
             if h > anchor_block.height {
                 None
             } else {
@@ -343,11 +416,11 @@ fn determine_tx_anchor(
     }
 }
 
-fn populate_with_outpoints<K>(
-    client: &Client,
-    anchor_block: BlockId,
-    update: &mut ElectrumUpdate<K, ConfirmationHeightAnchor>,
-    outpoints: &mut impl Iterator<Item = OutPoint>,
+fn populate_with_outpoints(
+    client: &impl ElectrumApi,
+    cps: &BTreeMap<u32, CheckPoint>,
+    relevant_txids: &mut RelevantTxids,
+    outpoints: impl IntoIterator<Item = OutPoint>,
 ) -> Result<HashMap<Txid, Transaction>, Error> {
     let mut full_txs = HashMap::new();
     for outpoint in outpoints {
@@ -394,9 +467,8 @@ fn populate_with_outpoints<K>(
                 }
             };
 
-            let anchor = determine_tx_anchor(anchor_block, res.height, res.tx_hash);
-
-            let tx_entry = update.graph_update.entry(res.tx_hash).or_default();
+            let anchor = determine_tx_anchor(cps, res.height, res.tx_hash);
+            let tx_entry = relevant_txids.0.entry(res.tx_hash).or_default();
             if let Some(anchor) = anchor {
                 tx_entry.insert(anchor);
             }
@@ -405,11 +477,11 @@ fn populate_with_outpoints<K>(
     Ok(full_txs)
 }
 
-fn populate_with_txids<K>(
-    client: &Client,
-    anchor_block: BlockId,
-    update: &mut ElectrumUpdate<K, ConfirmationHeightAnchor>,
-    txids: &mut impl Iterator<Item = Txid>,
+fn populate_with_txids(
+    client: &impl ElectrumApi,
+    cps: &BTreeMap<u32, CheckPoint>,
+    relevant_txids: &mut RelevantTxids,
+    txids: impl IntoIterator<Item = Txid>,
 ) -> Result<(), Error> {
     for txid in txids {
         let tx = match client.transaction_get(&txid) {
@@ -420,7 +492,7 @@ fn populate_with_txids<K>(
 
         let spk = tx
             .output
-            .get(0)
+            .first()
             .map(|txo| &txo.script_pubkey)
             .expect("tx must have an output");
 
@@ -429,11 +501,11 @@ fn populate_with_txids<K>(
             .into_iter()
             .find(|r| r.tx_hash == txid)
         {
-            Some(r) => determine_tx_anchor(anchor_block, r.height, txid),
+            Some(r) => determine_tx_anchor(cps, r.height, txid),
             None => continue,
         };
 
-        let tx_entry = update.graph_update.entry(txid).or_default();
+        let tx_entry = relevant_txids.0.entry(txid).or_default();
         if let Some(anchor) = anchor {
             tx_entry.insert(anchor);
         }
@@ -441,14 +513,14 @@ fn populate_with_txids<K>(
     Ok(())
 }
 
-fn populate_with_spks<K, I: Ord + Clone>(
-    client: &Client,
-    anchor_block: BlockId,
-    update: &mut ElectrumUpdate<K, ConfirmationHeightAnchor>,
-    spks: &mut impl Iterator<Item = (I, Script)>,
+fn populate_with_spks<I: Ord + Clone>(
+    client: &impl ElectrumApi,
+    cps: &BTreeMap<u32, CheckPoint>,
+    relevant_txids: &mut RelevantTxids,
+    spks: &mut impl Iterator<Item = (I, ScriptBuf)>,
     stop_gap: usize,
     batch_size: usize,
-) -> Result<BTreeMap<I, (Script, bool)>, Error> {
+) -> Result<BTreeMap<I, (ScriptBuf, bool)>, Error> {
     let mut unused_spk_count = 0_usize;
     let mut scanned_spks = BTreeMap::new();
 
@@ -460,7 +532,8 @@ fn populate_with_spks<K, I: Ord + Clone>(
             return Ok(scanned_spks);
         }
 
-        let spk_histories = client.batch_script_get_history(spks.iter().map(|(_, s)| s))?;
+        let spk_histories =
+            client.batch_script_get_history(spks.iter().map(|(_, s)| s.as_script()))?;
 
         for ((spk_index, spk), spk_history) in spks.into_iter().zip(spk_histories) {
             if spk_history.is_empty() {
@@ -476,8 +549,8 @@ fn populate_with_spks<K, I: Ord + Clone>(
             }
 
             for tx in spk_history {
-                let tx_entry = update.graph_update.entry(tx.tx_hash).or_default();
-                if let Some(anchor) = determine_tx_anchor(anchor_block, tx.height, tx.tx_hash) {
+                let tx_entry = relevant_txids.0.entry(tx.tx_hash).or_default();
+                if let Some(anchor) = determine_tx_anchor(cps, tx.height, tx.tx_hash) {
                     tx_entry.insert(anchor);
                 }
             }

crates/electrum/src/lib.rs

@@ -1,35 +1,30 @@
-//! This crate is used for updating structures of the [`bdk_chain`] crate with data from electrum.
+//! This crate is used for updating structures of [`bdk_chain`] with data from an Electrum server.
 //!
-//! The star of the show is the [`ElectrumExt::scan`] method, which scans for relevant blockchain
-//! data (via electrum) and outputs an [`ElectrumUpdate`].
+//! The two primary methods are [`ElectrumExt::sync`] and [`ElectrumExt::full_scan`]. In most cases
+//! [`ElectrumExt::sync`] is used to sync the transaction histories of scripts that the application
+//! cares about, for example the scripts for all the receive addresses of a Wallet's keychain that it
+//! has shown a user. [`ElectrumExt::full_scan`] is meant to be used when importing or restoring a
+//! keychain where the range of possibly used scripts is not known. In this case it is necessary to
+//! scan all keychain scripts until a number (the "stop gap") of unused scripts is discovered. For a
+//! sync or full scan the user receives relevant blockchain data and output updates for
+//! [`bdk_chain`] including [`RelevantTxids`].
 //!
-//! An [`ElectrumUpdate`] only includes `txid`s and no full transactions. The caller is responsible
-//! for obtaining full transactions before applying. This can be done with
-//! these steps:
+//! The [`RelevantTxids`] only includes `txid`s and not full transactions. The caller is responsible
+//! for obtaining full transactions before applying new data to their [`bdk_chain`]. This can be
+//! done with these steps:
 //!
-//! 1. Determine which full transactions are missing. The method [`missing_full_txs`] of
-//! [`ElectrumUpdate`] can be used.
+//! 1. Determine which full transactions are missing. Use [`RelevantTxids::missing_full_txs`].
 //!
-//! 2. Obtaining the full transactions. To do this via electrum, the method
-//! [`batch_transaction_get`] can be used.
+//! 2. Obtaining the full transactions. To do this via electrum use [`ElectrumApi::batch_transaction_get`].
 //!
-//! Refer to [`bdk_electrum_example`] for a complete example.
+//! Refer to [`example_electrum`] for a complete example.
 //!
-//! [`ElectrumClient::scan`]: ElectrumClient::scan
-//! [`missing_full_txs`]: ElectrumUpdate::missing_full_txs
-//! [`batch_transaction_get`]: ElectrumApi::batch_transaction_get
-//! [`bdk_electrum_example`]: https://github.com/LLFourn/bdk_core_staging/tree/master/bdk_electrum_example
+//! [`ElectrumApi::batch_transaction_get`]: electrum_client::ElectrumApi::batch_transaction_get
+//! [`example_electrum`]: https://github.com/bitcoindevkit/bdk/tree/master/example-crates/example_electrum
+
+#![warn(missing_docs)]
 
-use bdk_chain::bitcoin::BlockHash;
-use electrum_client::{Client, ElectrumApi, Error};
 mod electrum_ext;
 pub use bdk_chain;
 pub use electrum_client;
 pub use electrum_ext::*;
-
-fn get_tip(client: &Client) -> Result<(u32, BlockHash), Error> {
-    // TODO: unsubscribe when added to the client, or is there a better call to use here?
-    client
-        .block_headers_subscribe()
-        .map(|data| (data.height as u32, data.header.block_hash()))
-}

crates/esplora/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "bdk_esplora"
-version = "0.3.0"
+version = "0.9.0"
 edition = "2021"
 homepage = "https://bitcoindevkit.org"
 repository = "https://github.com/bitcoindevkit/bdk"
@@ -12,18 +12,23 @@ readme = "README.md"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-bdk_chain = { path = "../chain", version = "0.5.0", default-features = false, features = ["serde", "miniscript"] }
-esplora-client = { version = "0.5", default-features = false }
+bdk_chain = { path = "../chain", version = "0.11.0", default-features = false }
+esplora-client = { version = "0.6.0", default-features = false }
 async-trait = { version = "0.1.66", optional = true }
 futures = { version = "0.3.26", optional = true }
 
 # use these dependencies if you need to enable their /no-std features
-bitcoin = { version = "0.29", optional = true, default-features = false }
-miniscript = { version = "9.0.0", optional = true, default-features = false }
+bitcoin = { version = "0.30.0", optional = true, default-features = false }
+miniscript = { version = "10.0.0", optional = true, default-features = false }
+
+[target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies]
+electrsd = { version= "0.25.0", features = ["bitcoind_25_0", "esplora_a33e97e1", "legacy"] }
+tokio = { version = "1", features = ["rt", "rt-multi-thread", "macros"] }
 
 [features]
 default = ["std", "async-https", "blocking"]
 std = ["bdk_chain/std"]
 async = ["async-trait", "futures", "esplora-client/async"]
 async-https = ["async", "esplora-client/async-https"]
+async-https-rustls = ["async", "esplora-client/async-https-rustls"]
 blocking = ["esplora-client/blocking"]

crates/esplora/README.md

@@ -1,6 +1,6 @@
 # BDK Esplora
 
-BDK Esplora extends [`esplora_client`](crate::esplora_client) to update [`bdk_chain`] structures
+BDK Esplora extends [`esplora-client`] to update [`bdk_chain`] structures
 from an Esplora server.
 
 ## Usage
@@ -30,4 +30,7 @@ use bdk_esplora::EsploraExt;
 // use bdk_esplora::EsploraAsyncExt;
 ```
 
-For full examples, refer to [`example-crates/wallet_esplora`](https://github.com/bitcoindevkit/bdk/tree/master/example-crates/wallet_esplora) (blocking) and [`example-crates/wallet_esplora_async`](https://github.com/bitcoindevkit/bdk/tree/master/example-crates/wallet_esplora_async).
+For full examples, refer to [`example-crates/wallet_esplora_blocking`](https://github.com/bitcoindevkit/bdk/tree/master/example-crates/wallet_esplora_blocking) and [`example-crates/wallet_esplora_async`](https://github.com/bitcoindevkit/bdk/tree/master/example-crates/wallet_esplora_async).
+
+[`esplora-client`]: https://docs.rs/esplora-client/
+[`bdk_chain`]: https://docs.rs/bdk-chain/

crates/esplora/src/async_ext.rs

@@ -1,269 +1,320 @@
 use async_trait::async_trait;
+use bdk_chain::collections::btree_map;
 use bdk_chain::{
-    bitcoin::{BlockHash, OutPoint, Script, Txid},
+    bitcoin::{BlockHash, OutPoint, ScriptBuf, TxOut, Txid},
     collections::BTreeMap,
-    keychain::LocalUpdate,
-    BlockId, ConfirmationTimeAnchor,
+    local_chain::{self, CheckPoint},
+    BlockId, ConfirmationTimeHeightAnchor, TxGraph,
 };
-use esplora_client::{Error, OutputStatus, TxStatus};
+use esplora_client::TxStatus;
 use futures::{stream::FuturesOrdered, TryStreamExt};
 
-use crate::map_confirmation_time_anchor;
+use crate::anchor_from_status;
 
-/// Trait to extend [`esplora_client::AsyncClient`] functionality.
+/// [`esplora_client::Error`]
+type Error = Box<esplora_client::Error>;
+
+/// Trait to extend the functionality of [`esplora_client::AsyncClient`].
 ///
-/// This is the async version of [`EsploraExt`]. Refer to
-/// [crate-level documentation] for more.
+/// Refer to [crate-level documentation] for more.
 ///
-/// [`EsploraExt`]: crate::EsploraExt
 /// [crate-level documentation]: crate
 #[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
 #[cfg_attr(not(target_arch = "wasm32"), async_trait)]
 pub trait EsploraAsyncExt {
-    /// Scan the blockchain (via esplora) for the data specified and returns a
-    /// [`LocalUpdate<K, ConfirmationTimeAnchor>`].
+    /// Prepare a [`LocalChain`] update with blocks fetched from Esplora.
     ///
-    /// - `local_chain`: the most recent block hashes present locally
-    /// - `keychain_spks`: keychains that we want to scan transactions for
-    /// - `txids`: transactions for which we want updated [`ConfirmationTimeAnchor`]s
-    /// - `outpoints`: transactions associated with these outpoints (residing, spending) that we
-    ///     want to included in the update
+    /// * `local_tip` is the previous tip of [`LocalChain::tip`].
+    /// * `request_heights` is the block heights that we are interested in fetching from Esplora.
     ///
-    /// The scan for each keychain stops after a gap of `stop_gap` script pubkeys with no associated
+    /// The result of this method can be applied to [`LocalChain::apply_update`].
+    ///
+    /// ## Consistency
+    ///
+    /// The chain update returned is guaranteed to be consistent as long as there is not a *large* re-org
+    /// during the call. The size of re-org we can tollerate is server dependent but will be at
+    /// least 10.
+    ///
+    /// [`LocalChain`]: bdk_chain::local_chain::LocalChain
+    /// [`LocalChain::tip`]: bdk_chain::local_chain::LocalChain::tip
+    /// [`LocalChain::apply_update`]: bdk_chain::local_chain::LocalChain::apply_update
+    async fn update_local_chain(
+        &self,
+        local_tip: CheckPoint,
+        request_heights: impl IntoIterator<IntoIter = impl Iterator<Item = u32> + Send> + Send,
+    ) -> Result<local_chain::Update, Error>;
+
+    /// Full scan the keychain scripts specified with the blockchain (via an Esplora client) and
+    /// returns a [`TxGraph`] and a map of last active indices.
+    ///
+    /// * `keychain_spks`: keychains that we want to scan transactions for
+    ///
+    /// The full scan for each keychain stops after a gap of `stop_gap` script pubkeys with no associated
     /// transactions. `parallel_requests` specifies the max number of HTTP requests to make in
     /// parallel.
-    #[allow(clippy::result_large_err)] // FIXME
-    async fn scan<K: Ord + Clone + Send>(
+    async fn full_scan<K: Ord + Clone + Send>(
         &self,
-        local_chain: &BTreeMap<u32, BlockHash>,
         keychain_spks: BTreeMap<
             K,
-            impl IntoIterator<IntoIter = impl Iterator<Item = (u32, Script)> + Send> + Send,
+            impl IntoIterator<IntoIter = impl Iterator<Item = (u32, ScriptBuf)> + Send> + Send,
         >,
-        txids: impl IntoIterator<IntoIter = impl Iterator<Item = Txid> + Send> + Send,
-        outpoints: impl IntoIterator<IntoIter = impl Iterator<Item = OutPoint> + Send> + Send,
         stop_gap: usize,
         parallel_requests: usize,
-    ) -> Result<LocalUpdate<K, ConfirmationTimeAnchor>, Error>;
+    ) -> Result<(TxGraph<ConfirmationTimeHeightAnchor>, BTreeMap<K, u32>), Error>;
 
-    /// Convenience method to call [`scan`] without requiring a keychain.
+    /// Sync a set of scripts with the blockchain (via an Esplora client) for the data
+    /// specified and return a [`TxGraph`].
     ///
-    /// [`scan`]: EsploraAsyncExt::scan
-    #[allow(clippy::result_large_err)] // FIXME
-    async fn scan_without_keychain(
+    /// * `misc_spks`: scripts that we want to sync transactions for
+    /// * `txids`: transactions for which we want updated [`ConfirmationTimeHeightAnchor`]s
+    /// * `outpoints`: transactions associated with these outpoints (residing, spending) that we
+    ///     want to include in the update
+    ///
+    /// If the scripts to sync are unknown, such as when restoring or importing a keychain that
+    /// may include scripts that have been used, use [`full_scan`] with the keychain.
+    ///
+    /// [`full_scan`]: EsploraAsyncExt::full_scan
+    async fn sync(
         &self,
-        local_chain: &BTreeMap<u32, BlockHash>,
-        misc_spks: impl IntoIterator<IntoIter = impl Iterator<Item = Script> + Send> + Send,
+        misc_spks: impl IntoIterator<IntoIter = impl Iterator<Item = ScriptBuf> + Send> + Send,
         txids: impl IntoIterator<IntoIter = impl Iterator<Item = Txid> + Send> + Send,
         outpoints: impl IntoIterator<IntoIter = impl Iterator<Item = OutPoint> + Send> + Send,
         parallel_requests: usize,
-    ) -> Result<LocalUpdate<(), ConfirmationTimeAnchor>, Error> {
-        self.scan(
-            local_chain,
-            [(
-                (),
-                misc_spks
-                    .into_iter()
-                    .enumerate()
-                    .map(|(i, spk)| (i as u32, spk)),
-            )]
-            .into(),
-            txids,
-            outpoints,
-            usize::MAX,
-            parallel_requests,
-        )
-        .await
-    }
+    ) -> Result<TxGraph<ConfirmationTimeHeightAnchor>, Error>;
 }
 
 #[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
 #[cfg_attr(not(target_arch = "wasm32"), async_trait)]
 impl EsploraAsyncExt for esplora_client::AsyncClient {
-    #[allow(clippy::result_large_err)] // FIXME
-    async fn scan<K: Ord + Clone + Send>(
+    async fn update_local_chain(
         &self,
-        local_chain: &BTreeMap<u32, BlockHash>,
-        keychain_spks: BTreeMap<
-            K,
-            impl IntoIterator<IntoIter = impl Iterator<Item = (u32, Script)> + Send> + Send,
-        >,
-        txids: impl IntoIterator<IntoIter = impl Iterator<Item = Txid> + Send> + Send,
-        outpoints: impl IntoIterator<IntoIter = impl Iterator<Item = OutPoint> + Send> + Send,
-        stop_gap: usize,
-        parallel_requests: usize,
-    ) -> Result<LocalUpdate<K, ConfirmationTimeAnchor>, Error> {
-        let parallel_requests = Ord::max(parallel_requests, 1);
+        local_tip: CheckPoint,
+        request_heights: impl IntoIterator<IntoIter = impl Iterator<Item = u32> + Send> + Send,
+    ) -> Result<local_chain::Update, Error> {
+        // Fetch latest N (server dependent) blocks from Esplora. The server guarantees these are
+        // consistent.
+        let mut fetched_blocks = self
+            .get_blocks(None)
+            .await?
+            .into_iter()
+            .map(|b| (b.time.height, b.id))
+            .collect::<BTreeMap<u32, BlockHash>>();
+        let new_tip_height = fetched_blocks
+            .keys()
+            .last()
+            .copied()
+            .expect("must have atleast one block");
 
-        let (mut update, tip_at_start) = loop {
-            let mut update = LocalUpdate::<K, ConfirmationTimeAnchor>::default();
+        // Fetch blocks of heights that the caller is interested in, skipping blocks that are
+        // already fetched when constructing `fetched_blocks`.
+        for height in request_heights {
+            // do not fetch blocks higher than remote tip
+            if height > new_tip_height {
+                continue;
+            }
+            // only fetch what is missing
+            if let btree_map::Entry::Vacant(entry) = fetched_blocks.entry(height) {
+                // ❗The return value of `get_block_hash` is not strictly guaranteed to be consistent
+                // with the chain at the time of `get_blocks` above (there could have been a deep
+                // re-org). Since `get_blocks` returns 10 (or so) blocks we are assuming that it's
+                // not possible to have a re-org deeper than that.
+                entry.insert(self.get_block_hash(height).await?);
+            }
+        }
 
-            for (&height, &original_hash) in local_chain.iter().rev() {
-                let update_block_id = BlockId {
-                    height,
-                    hash: self.get_block_hash(height).await?,
-                };
-                let _ = update
-                    .chain
-                    .insert_block(update_block_id)
-                    .expect("cannot repeat height here");
-                if update_block_id.hash == original_hash {
-                    break;
-                }
+        // Ensure `fetched_blocks` can create an update that connects with the original chain by
+        // finding a "Point of Agreement".
+        for (height, local_hash) in local_tip.iter().map(|cp| (cp.height(), cp.hash())) {
+            if height > new_tip_height {
+                continue;
             }
 
-            let tip_at_start = BlockId {
-                height: self.get_height().await?,
-                hash: self.get_tip_hash().await?,
+            let fetched_hash = match fetched_blocks.entry(height) {
+                btree_map::Entry::Occupied(entry) => *entry.get(),
+                btree_map::Entry::Vacant(entry) => {
+                    *entry.insert(self.get_block_hash(height).await?)
+                }
             };
 
-            if update.chain.insert_block(tip_at_start).is_ok() {
-                break (update, tip_at_start);
+            // We have found point of agreement so the update will connect!
+            if fetched_hash == local_hash {
+                break;
             }
-        };
+        }
+
+        Ok(local_chain::Update {
+            tip: CheckPoint::from_block_ids(fetched_blocks.into_iter().map(BlockId::from))
+                .expect("must be in height order"),
+            introduce_older_blocks: true,
+        })
+    }
+
+    async fn full_scan<K: Ord + Clone + Send>(
+        &self,
+        keychain_spks: BTreeMap<
+            K,
+            impl IntoIterator<IntoIter = impl Iterator<Item = (u32, ScriptBuf)> + Send> + Send,
+        >,
+        stop_gap: usize,
+        parallel_requests: usize,
+    ) -> Result<(TxGraph<ConfirmationTimeHeightAnchor>, BTreeMap<K, u32>), Error> {
+        type TxsOfSpkIndex = (u32, Vec<esplora_client::Tx>);
+        let parallel_requests = Ord::max(parallel_requests, 1);
+        let mut graph = TxGraph::<ConfirmationTimeHeightAnchor>::default();
+        let mut last_active_indexes = BTreeMap::<K, u32>::new();
 
         for (keychain, spks) in keychain_spks {
             let mut spks = spks.into_iter();
-            let mut last_active_index = None;
-            let mut empty_scripts = 0;
-            type IndexWithTxs = (u32, Vec<esplora_client::Tx>);
+            let mut last_index = Option::<u32>::None;
+            let mut last_active_index = Option::<u32>::None;
 
             loop {
-                let futures = (0..parallel_requests)
-                    .filter_map(|_| {
-                        let (index, script) = spks.next()?;
+                let handles = spks
+                    .by_ref()
+                    .take(parallel_requests)
+                    .map(|(spk_index, spk)| {
                         let client = self.clone();
-                        Some(async move {
-                            let mut related_txs = client.scripthash_txs(&script, None).await?;
-
-                            let n_confirmed =
-                                related_txs.iter().filter(|tx| tx.status.confirmed).count();
-                            // esplora pages on 25 confirmed transactions. If there are 25 or more we
-                            // keep requesting to see if there's more.
-                            if n_confirmed >= 25 {
-                                loop {
-                                    let new_related_txs = client
-                                        .scripthash_txs(
-                                            &script,
-                                            Some(related_txs.last().unwrap().txid),
-                                        )
-                                        .await?;
-                                    let n = new_related_txs.len();
-                                    related_txs.extend(new_related_txs);
-                                    // we've reached the end
-                                    if n < 25 {
-                                        break;
-                                    }
+                        async move {
+                            let mut last_seen = None;
+                            let mut spk_txs = Vec::new();
+                            loop {
+                                let txs = client.scripthash_txs(&spk, last_seen).await?;
+                                let tx_count = txs.len();
+                                last_seen = txs.last().map(|tx| tx.txid);
+                                spk_txs.extend(txs);
+                                if tx_count < 25 {
+                                    break Result::<_, Error>::Ok((spk_index, spk_txs));
                                 }
                             }
-
-                            Result::<_, esplora_client::Error>::Ok((index, related_txs))
-                        })
+                        }
                     })
                     .collect::<FuturesOrdered<_>>();
 
-                let n_futures = futures.len();
+                if handles.is_empty() {
+                    break;
+                }
 
-                for (index, related_txs) in futures.try_collect::<Vec<IndexWithTxs>>().await? {
-                    if related_txs.is_empty() {
-                        empty_scripts += 1;
-                    } else {
+                for (index, txs) in handles.try_collect::<Vec<TxsOfSpkIndex>>().await? {
+                    last_index = Some(index);
+                    if !txs.is_empty() {
                         last_active_index = Some(index);
-                        empty_scripts = 0;
                     }
-                    for tx in related_txs {
-                        let anchor = map_confirmation_time_anchor(&tx.status, tip_at_start);
+                    for tx in txs {
+                        let _ = graph.insert_tx(tx.to_tx());
+                        if let Some(anchor) = anchor_from_status(&tx.status) {
+                            let _ = graph.insert_anchor(tx.txid, anchor);
+                        }
 
-                        let _ = update.graph.insert_tx(tx.to_tx());
-                        if let Some(anchor) = anchor {
-                            let _ = update.graph.insert_anchor(tx.txid, anchor);
+                        let previous_outputs = tx.vin.iter().filter_map(|vin| {
+                            let prevout = vin.prevout.as_ref()?;
+                            Some((
+                                OutPoint {
+                                    txid: vin.txid,
+                                    vout: vin.vout,
+                                },
+                                TxOut {
+                                    script_pubkey: prevout.scriptpubkey.clone(),
+                                    value: prevout.value,
+                                },
+                            ))
+                        });
+
+                        for (outpoint, txout) in previous_outputs {
+                            let _ = graph.insert_txout(outpoint, txout);
                         }
                     }
                 }
 
-                if n_futures == 0 || empty_scripts >= stop_gap {
+                let last_index = last_index.expect("Must be set since handles wasn't empty.");
+                let past_gap_limit = if let Some(i) = last_active_index {
+                    last_index > i.saturating_add(stop_gap as u32)
+                } else {
+                    last_index >= stop_gap as u32
+                };
+                if past_gap_limit {
                     break;
                 }
             }
 
             if let Some(last_active_index) = last_active_index {
-                update.keychain.insert(keychain, last_active_index);
+                last_active_indexes.insert(keychain, last_active_index);
             }
         }
 
-        for txid in txids.into_iter() {
-            if update.graph.get_tx(txid).is_none() {
-                match self.get_tx(&txid).await? {
-                    Some(tx) => {
-                        let _ = update.graph.insert_tx(tx);
-                    }
-                    None => continue,
-                }
+        Ok((graph, last_active_indexes))
+    }
+
+    async fn sync(
+        &self,
+        misc_spks: impl IntoIterator<IntoIter = impl Iterator<Item = ScriptBuf> + Send> + Send,
+        txids: impl IntoIterator<IntoIter = impl Iterator<Item = Txid> + Send> + Send,
+        outpoints: impl IntoIterator<IntoIter = impl Iterator<Item = OutPoint> + Send> + Send,
+        parallel_requests: usize,
+    ) -> Result<TxGraph<ConfirmationTimeHeightAnchor>, Error> {
+        let mut graph = self
+            .full_scan(
+                [(
+                    (),
+                    misc_spks
+                        .into_iter()
+                        .enumerate()
+                        .map(|(i, spk)| (i as u32, spk)),
+                )]
+                .into(),
+                usize::MAX,
+                parallel_requests,
+            )
+            .await
+            .map(|(g, _)| g)?;
+
+        let mut txids = txids.into_iter();
+        loop {
+            let handles = txids
+                .by_ref()
+                .take(parallel_requests)
+                .filter(|&txid| graph.get_tx(txid).is_none())
+                .map(|txid| {
+                    let client = self.clone();
+                    async move { client.get_tx_status(&txid).await.map(|s| (txid, s)) }
+                })
+                .collect::<FuturesOrdered<_>>();
+
+            if handles.is_empty() {
+                break;
             }
-            match self.get_tx_status(&txid).await? {
-                tx_status if tx_status.confirmed => {
-                    if let Some(anchor) = map_confirmation_time_anchor(&tx_status, tip_at_start) {
-                        let _ = update.graph.insert_anchor(txid, anchor);
-                    }
+
+            for (txid, status) in handles.try_collect::<Vec<(Txid, TxStatus)>>().await? {
+                if let Some(anchor) = anchor_from_status(&status) {
+                    let _ = graph.insert_anchor(txid, anchor);
                 }
-                _ => continue,
             }
         }
 
         for op in outpoints.into_iter() {
-            let mut op_txs = Vec::with_capacity(2);
-            if let (
-                Some(tx),
-                tx_status @ TxStatus {
-                    confirmed: true, ..
-                },
-            ) = (
-                self.get_tx(&op.txid).await?,
-                self.get_tx_status(&op.txid).await?,
-            ) {
-                op_txs.push((tx, tx_status));
-                if let Some(OutputStatus {
-                    txid: Some(txid),
-                    status: Some(spend_status),
-                    ..
-                }) = self.get_output_status(&op.txid, op.vout as _).await?
-                {
-                    if let Some(spend_tx) = self.get_tx(&txid).await? {
-                        op_txs.push((spend_tx, spend_status));
+            if graph.get_tx(op.txid).is_none() {
+                if let Some(tx) = self.get_tx(&op.txid).await? {
+                    let _ = graph.insert_tx(tx);
+                }
+                let status = self.get_tx_status(&op.txid).await?;
+                if let Some(anchor) = anchor_from_status(&status) {
+                    let _ = graph.insert_anchor(op.txid, anchor);
+                }
+            }
+
+            if let Some(op_status) = self.get_output_status(&op.txid, op.vout as _).await? {
+                if let Some(txid) = op_status.txid {
+                    if graph.get_tx(txid).is_none() {
+                        if let Some(tx) = self.get_tx(&txid).await? {
+                            let _ = graph.insert_tx(tx);
+                        }
+                        let status = self.get_tx_status(&txid).await?;
+                        if let Some(anchor) = anchor_from_status(&status) {
+                            let _ = graph.insert_anchor(txid, anchor);
+                        }
                     }
                 }
             }
-
-            for (tx, status) in op_txs {
-                let txid = tx.txid();
-                let anchor = map_confirmation_time_anchor(&status, tip_at_start);
-
-                let _ = update.graph.insert_tx(tx);
-                if let Some(anchor) = anchor {
-                    let _ = update.graph.insert_anchor(txid, anchor);
-                }
-            }
         }
-
-        if tip_at_start.hash != self.get_block_hash(tip_at_start.height).await? {
-            // A reorg occurred, so let's find out where all the txids we found are now in the chain
-            let txids_found = update
-                .graph
-                .full_txs()
-                .map(|tx_node| tx_node.txid)
-                .collect::<Vec<_>>();
-            update.chain = EsploraAsyncExt::scan_without_keychain(
-                self,
-                local_chain,
-                [],
-                txids_found,
-                [],
-                parallel_requests,
-            )
-            .await?
-            .chain;
-        }
-
-        Ok(update)
+        Ok(graph)
     }
 }

crates/esplora/src/blocking_ext.rs

@@ -1,251 +1,317 @@
-use bdk_chain::bitcoin::{BlockHash, OutPoint, Script, Txid};
+use std::thread::JoinHandle;
+
+use bdk_chain::collections::btree_map;
 use bdk_chain::collections::BTreeMap;
-use bdk_chain::BlockId;
-use bdk_chain::{keychain::LocalUpdate, ConfirmationTimeAnchor};
-use esplora_client::{Error, OutputStatus, TxStatus};
+use bdk_chain::{
+    bitcoin::{BlockHash, OutPoint, ScriptBuf, TxOut, Txid},
+    local_chain::{self, CheckPoint},
+    BlockId, ConfirmationTimeHeightAnchor, TxGraph,
+};
+use esplora_client::TxStatus;
 
-use crate::map_confirmation_time_anchor;
+use crate::anchor_from_status;
 
-/// Trait to extend [`esplora_client::BlockingClient`] functionality.
+/// [`esplora_client::Error`]
+type Error = Box<esplora_client::Error>;
+
+/// Trait to extend the functionality of [`esplora_client::BlockingClient`].
 ///
 /// Refer to [crate-level documentation] for more.
 ///
 /// [crate-level documentation]: crate
 pub trait EsploraExt {
-    /// Scan the blockchain (via esplora) for the data specified and returns a
-    /// [`LocalUpdate<K, ConfirmationTimeAnchor>`].
+    /// Prepare a [`LocalChain`] update with blocks fetched from Esplora.
     ///
-    /// - `local_chain`: the most recent block hashes present locally
-    /// - `keychain_spks`: keychains that we want to scan transactions for
-    /// - `txids`: transactions for which we want updated [`ConfirmationTimeAnchor`]s
-    /// - `outpoints`: transactions associated with these outpoints (residing, spending) that we
-    ///     want to included in the update
+    /// * `local_tip` is the previous tip of [`LocalChain::tip`].
+    /// * `request_heights` is the block heights that we are interested in fetching from Esplora.
     ///
-    /// The scan for each keychain stops after a gap of `stop_gap` script pubkeys with no associated
+    /// The result of this method can be applied to [`LocalChain::apply_update`].
+    ///
+    /// ## Consistency
+    ///
+    /// The chain update returned is guaranteed to be consistent as long as there is not a *large* re-org
+    /// during the call. The size of re-org we can tollerate is server dependent but will be at
+    /// least 10.
+    ///
+    /// [`LocalChain`]: bdk_chain::local_chain::LocalChain
+    /// [`LocalChain::tip`]: bdk_chain::local_chain::LocalChain::tip
+    /// [`LocalChain::apply_update`]: bdk_chain::local_chain::LocalChain::apply_update
+    fn update_local_chain(
+        &self,
+        local_tip: CheckPoint,
+        request_heights: impl IntoIterator<Item = u32>,
+    ) -> Result<local_chain::Update, Error>;
+
+    /// Full scan the keychain scripts specified with the blockchain (via an Esplora client) and
+    /// returns a [`TxGraph`] and a map of last active indices.
+    ///
+    /// * `keychain_spks`: keychains that we want to scan transactions for
+    ///
+    /// The full scan for each keychain stops after a gap of `stop_gap` script pubkeys with no associated
     /// transactions. `parallel_requests` specifies the max number of HTTP requests to make in
     /// parallel.
-    #[allow(clippy::result_large_err)] // FIXME
-    fn scan<K: Ord + Clone>(
+    fn full_scan<K: Ord + Clone>(
         &self,
-        local_chain: &BTreeMap<u32, BlockHash>,
-        keychain_spks: BTreeMap<K, impl IntoIterator<Item = (u32, Script)>>,
-        txids: impl IntoIterator<Item = Txid>,
-        outpoints: impl IntoIterator<Item = OutPoint>,
+        keychain_spks: BTreeMap<K, impl IntoIterator<Item = (u32, ScriptBuf)>>,
         stop_gap: usize,
         parallel_requests: usize,
-    ) -> Result<LocalUpdate<K, ConfirmationTimeAnchor>, Error>;
+    ) -> Result<(TxGraph<ConfirmationTimeHeightAnchor>, BTreeMap<K, u32>), Error>;
 
-    /// Convenience method to call [`scan`] without requiring a keychain.
+    /// Sync a set of scripts with the blockchain (via an Esplora client) for the data
+    /// specified and return a [`TxGraph`].
     ///
-    /// [`scan`]: EsploraExt::scan
-    #[allow(clippy::result_large_err)] // FIXME
-    fn scan_without_keychain(
+    /// * `misc_spks`: scripts that we want to sync transactions for
+    /// * `txids`: transactions for which we want updated [`ConfirmationTimeHeightAnchor`]s
+    /// * `outpoints`: transactions associated with these outpoints (residing, spending) that we
+    ///     want to include in the update
+    ///
+    /// If the scripts to sync are unknown, such as when restoring or importing a keychain that
+    /// may include scripts that have been used, use [`full_scan`] with the keychain.
+    ///
+    /// [`full_scan`]: EsploraExt::full_scan
+    fn sync(
         &self,
-        local_chain: &BTreeMap<u32, BlockHash>,
-        misc_spks: impl IntoIterator<Item = Script>,
+        misc_spks: impl IntoIterator<Item = ScriptBuf>,
         txids: impl IntoIterator<Item = Txid>,
         outpoints: impl IntoIterator<Item = OutPoint>,
         parallel_requests: usize,
-    ) -> Result<LocalUpdate<(), ConfirmationTimeAnchor>, Error> {
-        self.scan(
-            local_chain,
-            [(
-                (),
-                misc_spks
-                    .into_iter()
-                    .enumerate()
-                    .map(|(i, spk)| (i as u32, spk)),
-            )]
-            .into(),
-            txids,
-            outpoints,
-            usize::MAX,
-            parallel_requests,
-        )
-    }
+    ) -> Result<TxGraph<ConfirmationTimeHeightAnchor>, Error>;
 }
 
 impl EsploraExt for esplora_client::BlockingClient {
-    fn scan<K: Ord + Clone>(
+    fn update_local_chain(
         &self,
-        local_chain: &BTreeMap<u32, BlockHash>,
-        keychain_spks: BTreeMap<K, impl IntoIterator<Item = (u32, Script)>>,
-        txids: impl IntoIterator<Item = Txid>,
-        outpoints: impl IntoIterator<Item = OutPoint>,
-        stop_gap: usize,
-        parallel_requests: usize,
-    ) -> Result<LocalUpdate<K, ConfirmationTimeAnchor>, Error> {
-        let parallel_requests = Ord::max(parallel_requests, 1);
+        local_tip: CheckPoint,
+        request_heights: impl IntoIterator<Item = u32>,
+    ) -> Result<local_chain::Update, Error> {
+        // Fetch latest N (server dependent) blocks from Esplora. The server guarantees these are
+        // consistent.
+        let mut fetched_blocks = self
+            .get_blocks(None)?
+            .into_iter()
+            .map(|b| (b.time.height, b.id))
+            .collect::<BTreeMap<u32, BlockHash>>();
+        let new_tip_height = fetched_blocks
+            .keys()
+            .last()
+            .copied()
+            .expect("must atleast have one block");
 
-        let (mut update, tip_at_start) = loop {
-            let mut update = LocalUpdate::<K, ConfirmationTimeAnchor>::default();
+        // Fetch blocks of heights that the caller is interested in, skipping blocks that are
+        // already fetched when constructing `fetched_blocks`.
+        for height in request_heights {
+            // do not fetch blocks higher than remote tip
+            if height > new_tip_height {
+                continue;
+            }
+            // only fetch what is missing
+            if let btree_map::Entry::Vacant(entry) = fetched_blocks.entry(height) {
+                // ❗The return value of `get_block_hash` is not strictly guaranteed to be consistent
+                // with the chain at the time of `get_blocks` above (there could have been a deep
+                // re-org). Since `get_blocks` returns 10 (or so) blocks we are assuming that it's
+                // not possible to have a re-org deeper than that.
+                entry.insert(self.get_block_hash(height)?);
+            }
+        }
 
-            for (&height, &original_hash) in local_chain.iter().rev() {
-                let update_block_id = BlockId {
-                    height,
-                    hash: self.get_block_hash(height)?,
-                };
-                let _ = update
-                    .chain
-                    .insert_block(update_block_id)
-                    .expect("cannot repeat height here");
-                if update_block_id.hash == original_hash {
-                    break;
-                }
+        // Ensure `fetched_blocks` can create an update that connects with the original chain by
+        // finding a "Point of Agreement".
+        for (height, local_hash) in local_tip.iter().map(|cp| (cp.height(), cp.hash())) {
+            if height > new_tip_height {
+                continue;
             }
 
-            let tip_at_start = BlockId {
-                height: self.get_height()?,
-                hash: self.get_tip_hash()?,
+            let fetched_hash = match fetched_blocks.entry(height) {
+                btree_map::Entry::Occupied(entry) => *entry.get(),
+                btree_map::Entry::Vacant(entry) => *entry.insert(self.get_block_hash(height)?),
             };
 
-            if update.chain.insert_block(tip_at_start).is_ok() {
-                break (update, tip_at_start);
+            // We have found point of agreement so the update will connect!
+            if fetched_hash == local_hash {
+                break;
             }
-        };
+        }
+
+        Ok(local_chain::Update {
+            tip: CheckPoint::from_block_ids(fetched_blocks.into_iter().map(BlockId::from))
+                .expect("must be in height order"),
+            introduce_older_blocks: true,
+        })
+    }
+
+    fn full_scan<K: Ord + Clone>(
+        &self,
+        keychain_spks: BTreeMap<K, impl IntoIterator<Item = (u32, ScriptBuf)>>,
+        stop_gap: usize,
+        parallel_requests: usize,
+    ) -> Result<(TxGraph<ConfirmationTimeHeightAnchor>, BTreeMap<K, u32>), Error> {
+        type TxsOfSpkIndex = (u32, Vec<esplora_client::Tx>);
+        let parallel_requests = Ord::max(parallel_requests, 1);
+        let mut graph = TxGraph::<ConfirmationTimeHeightAnchor>::default();
+        let mut last_active_indexes = BTreeMap::<K, u32>::new();
 
         for (keychain, spks) in keychain_spks {
             let mut spks = spks.into_iter();
-            let mut last_active_index = None;
-            let mut empty_scripts = 0;
-            type IndexWithTxs = (u32, Vec<esplora_client::Tx>);
+            let mut last_index = Option::<u32>::None;
+            let mut last_active_index = Option::<u32>::None;
 
             loop {
-                let handles = (0..parallel_requests)
-                    .filter_map(
-                        |_| -> Option<std::thread::JoinHandle<Result<IndexWithTxs, _>>> {
-                            let (index, script) = spks.next()?;
+                let handles = spks
+                    .by_ref()
+                    .take(parallel_requests)
+                    .map(|(spk_index, spk)| {
+                        std::thread::spawn({
                             let client = self.clone();
-                            Some(std::thread::spawn(move || {
-                                let mut related_txs = client.scripthash_txs(&script, None)?;
-
-                                let n_confirmed =
-                                    related_txs.iter().filter(|tx| tx.status.confirmed).count();
-                                // esplora pages on 25 confirmed transactions. If there are 25 or more we
-                                // keep requesting to see if there's more.
-                                if n_confirmed >= 25 {
-                                    loop {
-                                        let new_related_txs = client.scripthash_txs(
-                                            &script,
-                                            Some(related_txs.last().unwrap().txid),
-                                        )?;
-                                        let n = new_related_txs.len();
-                                        related_txs.extend(new_related_txs);
-                                        // we've reached the end
-                                        if n < 25 {
-                                            break;
-                                        }
+                            move || -> Result<TxsOfSpkIndex, Error> {
+                                let mut last_seen = None;
+                                let mut spk_txs = Vec::new();
+                                loop {
+                                    let txs = client.scripthash_txs(&spk, last_seen)?;
+                                    let tx_count = txs.len();
+                                    last_seen = txs.last().map(|tx| tx.txid);
+                                    spk_txs.extend(txs);
+                                    if tx_count < 25 {
+                                        break Ok((spk_index, spk_txs));
                                     }
                                 }
+                            }
+                        })
+                    })
+                    .collect::<Vec<JoinHandle<Result<TxsOfSpkIndex, Error>>>>();
 
-                                Result::<_, esplora_client::Error>::Ok((index, related_txs))
-                            }))
-                        },
-                    )
-                    .collect::<Vec<_>>();
-
-                let n_handles = handles.len();
+                if handles.is_empty() {
+                    break;
+                }
 
                 for handle in handles {
-                    let (index, related_txs) = handle.join().unwrap()?; // TODO: don't unwrap
-                    if related_txs.is_empty() {
-                        empty_scripts += 1;
-                    } else {
+                    let (index, txs) = handle.join().expect("thread must not panic")?;
+                    last_index = Some(index);
+                    if !txs.is_empty() {
                         last_active_index = Some(index);
-                        empty_scripts = 0;
                     }
-                    for tx in related_txs {
-                        let anchor = map_confirmation_time_anchor(&tx.status, tip_at_start);
+                    for tx in txs {
+                        let _ = graph.insert_tx(tx.to_tx());
+                        if let Some(anchor) = anchor_from_status(&tx.status) {
+                            let _ = graph.insert_anchor(tx.txid, anchor);
+                        }
 
-                        let _ = update.graph.insert_tx(tx.to_tx());
-                        if let Some(anchor) = anchor {
-                            let _ = update.graph.insert_anchor(tx.txid, anchor);
+                        let previous_outputs = tx.vin.iter().filter_map(|vin| {
+                            let prevout = vin.prevout.as_ref()?;
+                            Some((
+                                OutPoint {
+                                    txid: vin.txid,
+                                    vout: vin.vout,
+                                },
+                                TxOut {
+                                    script_pubkey: prevout.scriptpubkey.clone(),
+                                    value: prevout.value,
+                                },
+                            ))
+                        });
+
+                        for (outpoint, txout) in previous_outputs {
+                            let _ = graph.insert_txout(outpoint, txout);
                         }
                     }
                 }
 
-                if n_handles == 0 || empty_scripts >= stop_gap {
+                let last_index = last_index.expect("Must be set since handles wasn't empty.");
+                let past_gap_limit = if let Some(i) = last_active_index {
+                    last_index > i.saturating_add(stop_gap as u32)
+                } else {
+                    last_index >= stop_gap as u32
+                };
+                if past_gap_limit {
                     break;
                 }
             }
 
             if let Some(last_active_index) = last_active_index {
-                update.keychain.insert(keychain, last_active_index);
+                last_active_indexes.insert(keychain, last_active_index);
             }
         }
 
-        for txid in txids.into_iter() {
-            if update.graph.get_tx(txid).is_none() {
-                match self.get_tx(&txid)? {
-                    Some(tx) => {
-                        let _ = update.graph.insert_tx(tx);
-                    }
-                    None => continue,
-                }
-            }
-            match self.get_tx_status(&txid)? {
-                tx_status @ TxStatus {
-                    confirmed: true, ..
-                } => {
-                    if let Some(anchor) = map_confirmation_time_anchor(&tx_status, tip_at_start) {
-                        let _ = update.graph.insert_anchor(txid, anchor);
-                    }
-                }
-                _ => continue,
-            }
-        }
+        Ok((graph, last_active_indexes))
+    }
 
-        for op in outpoints.into_iter() {
-            let mut op_txs = Vec::with_capacity(2);
-            if let (
-                Some(tx),
-                tx_status @ TxStatus {
-                    confirmed: true, ..
-                },
-            ) = (self.get_tx(&op.txid)?, self.get_tx_status(&op.txid)?)
-            {
-                op_txs.push((tx, tx_status));
-                if let Some(OutputStatus {
-                    txid: Some(txid),
-                    status: Some(spend_status),
-                    ..
-                }) = self.get_output_status(&op.txid, op.vout as _)?
-                {
-                    if let Some(spend_tx) = self.get_tx(&txid)? {
-                        op_txs.push((spend_tx, spend_status));
-                    }
-                }
-            }
-
-            for (tx, status) in op_txs {
-                let txid = tx.txid();
-                let anchor = map_confirmation_time_anchor(&status, tip_at_start);
-
-                let _ = update.graph.insert_tx(tx);
-                if let Some(anchor) = anchor {
-                    let _ = update.graph.insert_anchor(txid, anchor);
-                }
-            }
-        }
-
-        if tip_at_start.hash != self.get_block_hash(tip_at_start.height)? {
-            // A reorg occurred, so let's find out where all the txids we found are now in the chain
-            let txids_found = update
-                .graph
-                .full_txs()
-                .map(|tx_node| tx_node.txid)
-                .collect::<Vec<_>>();
-            update.chain = EsploraExt::scan_without_keychain(
-                self,
-                local_chain,
-                [],
-                txids_found,
-                [],
+    fn sync(
+        &self,
+        misc_spks: impl IntoIterator<Item = ScriptBuf>,
+        txids: impl IntoIterator<Item = Txid>,
+        outpoints: impl IntoIterator<Item = OutPoint>,
+        parallel_requests: usize,
+    ) -> Result<TxGraph<ConfirmationTimeHeightAnchor>, Error> {
+        let mut graph = self
+            .full_scan(
+                [(
+                    (),
+                    misc_spks
+                        .into_iter()
+                        .enumerate()
+                        .map(|(i, spk)| (i as u32, spk)),
+                )]
+                .into(),
+                usize::MAX,
                 parallel_requests,
-            )?
-            .chain;
+            )
+            .map(|(g, _)| g)?;
+
+        let mut txids = txids.into_iter();
+        loop {
+            let handles = txids
+                .by_ref()
+                .take(parallel_requests)
+                .filter(|&txid| graph.get_tx(txid).is_none())
+                .map(|txid| {
+                    std::thread::spawn({
+                        let client = self.clone();
+                        move || {
+                            client
+                                .get_tx_status(&txid)
+                                .map_err(Box::new)
+                                .map(|s| (txid, s))
+                        }
+                    })
+                })
+                .collect::<Vec<JoinHandle<Result<(Txid, TxStatus), Error>>>>();
+
+            if handles.is_empty() {
+                break;
+            }
+
+            for handle in handles {
+                let (txid, status) = handle.join().expect("thread must not panic")?;
+                if let Some(anchor) = anchor_from_status(&status) {
+                    let _ = graph.insert_anchor(txid, anchor);
+                }
+            }
         }
 
-        Ok(update)
+        for op in outpoints {
+            if graph.get_tx(op.txid).is_none() {
+                if let Some(tx) = self.get_tx(&op.txid)? {
+                    let _ = graph.insert_tx(tx);
+                }
+                let status = self.get_tx_status(&op.txid)?;
+                if let Some(anchor) = anchor_from_status(&status) {
+                    let _ = graph.insert_anchor(op.txid, anchor);
+                }
+            }
+
+            if let Some(op_status) = self.get_output_status(&op.txid, op.vout as _)? {
+                if let Some(txid) = op_status.txid {
+                    if graph.get_tx(txid).is_none() {
+                        if let Some(tx) = self.get_tx(&txid)? {
+                            let _ = graph.insert_tx(tx);
+                        }
+                        let status = self.get_tx_status(&txid)?;
+                        if let Some(anchor) = anchor_from_status(&status) {
+                            let _ = graph.insert_anchor(txid, anchor);
+                        }
+                    }
+                }
+            }
+        }
+        Ok(graph)
     }
 }

crates/esplora/src/lib.rs

@@ -1,5 +1,22 @@
 #![doc = include_str!("../README.md")]
-use bdk_chain::{BlockId, ConfirmationTimeAnchor};
+
+//! This crate is used for updating structures of [`bdk_chain`] with data from an Esplora server.
+//!
+//! The two primary methods are [`EsploraExt::sync`] and [`EsploraExt::full_scan`]. In most cases
+//! [`EsploraExt::sync`] is used to sync the transaction histories of scripts that the application
+//! cares about, for example the scripts for all the receive addresses of a Wallet's keychain that it
+//! has shown a user. [`EsploraExt::full_scan`] is meant to be used when importing or restoring a
+//! keychain where the range of possibly used scripts is not known. In this case it is necessary to
+//! scan all keychain scripts until a number (the "stop gap") of unused scripts is discovered. For a
+//! sync or full scan the user receives relevant blockchain data and output updates for [`bdk_chain`]
+//! via a new [`TxGraph`] to be appended to any existing [`TxGraph`] data.
+//!
+//! Refer to [`example_esplora`] for a complete example.
+//!
+//! [`TxGraph`]: bdk_chain::tx_graph::TxGraph
+//! [`example_esplora`]: https://github.com/bitcoindevkit/bdk/tree/master/example-crates/example_esplora
+
+use bdk_chain::{BlockId, ConfirmationTimeHeightAnchor};
 use esplora_client::TxStatus;
 
 pub use esplora_client;
@@ -14,16 +31,20 @@ mod async_ext;
 #[cfg(feature = "async")]
 pub use async_ext::*;
 
-pub(crate) fn map_confirmation_time_anchor(
-    tx_status: &TxStatus,
-    tip_at_start: BlockId,
-) -> Option<ConfirmationTimeAnchor> {
-    match (tx_status.block_time, tx_status.block_height) {
-        (Some(confirmation_time), Some(confirmation_height)) => Some(ConfirmationTimeAnchor {
-            anchor_block: tip_at_start,
-            confirmation_height,
-            confirmation_time,
-        }),
-        _ => None,
+fn anchor_from_status(status: &TxStatus) -> Option<ConfirmationTimeHeightAnchor> {
+    if let TxStatus {
+        block_height: Some(height),
+        block_hash: Some(hash),
+        block_time: Some(time),
+        ..
+    } = status.clone()
+    {
+        Some(ConfirmationTimeHeightAnchor {
+            anchor_block: BlockId { height, hash },
+            confirmation_height: height,
+            confirmation_time: time,
+        })
+    } else {
+        None
     }
 }

crates/esplora/tests/async_ext.rs

@@ -0,0 +1,228 @@
+use bdk_esplora::EsploraAsyncExt;
+use electrsd::bitcoind::bitcoincore_rpc::RpcApi;
+use electrsd::bitcoind::{self, anyhow, BitcoinD};
+use electrsd::{Conf, ElectrsD};
+use esplora_client::{self, AsyncClient, Builder};
+use std::collections::{BTreeMap, HashSet};
+use std::str::FromStr;
+use std::thread::sleep;
+use std::time::Duration;
+
+use bdk_chain::bitcoin::{Address, Amount, BlockHash, Txid};
+
+struct TestEnv {
+    bitcoind: BitcoinD,
+    #[allow(dead_code)]
+    electrsd: ElectrsD,
+    client: AsyncClient,
+}
+
+impl TestEnv {
+    fn new() -> Result<Self, anyhow::Error> {
+        let bitcoind_exe =
+            bitcoind::downloaded_exe_path().expect("bitcoind version feature must be enabled");
+        let bitcoind = BitcoinD::new(bitcoind_exe).unwrap();
+
+        let mut electrs_conf = Conf::default();
+        electrs_conf.http_enabled = true;
+        let electrs_exe =
+            electrsd::downloaded_exe_path().expect("electrs version feature must be enabled");
+        let electrsd = ElectrsD::with_conf(electrs_exe, &bitcoind, &electrs_conf)?;
+
+        let base_url = format!("http://{}", &electrsd.esplora_url.clone().unwrap());
+        let client = Builder::new(base_url.as_str()).build_async()?;
+
+        Ok(Self {
+            bitcoind,
+            electrsd,
+            client,
+        })
+    }
+
+    fn mine_blocks(
+        &self,
+        count: usize,
+        address: Option<Address>,
+    ) -> anyhow::Result<Vec<BlockHash>> {
+        let coinbase_address = match address {
+            Some(address) => address,
+            None => self
+                .bitcoind
+                .client
+                .get_new_address(None, None)?
+                .assume_checked(),
+        };
+        let block_hashes = self
+            .bitcoind
+            .client
+            .generate_to_address(count as _, &coinbase_address)?;
+        Ok(block_hashes)
+    }
+}
+
+#[tokio::test]
+pub async fn test_update_tx_graph_without_keychain() -> anyhow::Result<()> {
+    let env = TestEnv::new()?;
+    let receive_address0 =
+        Address::from_str("bcrt1qc6fweuf4xjvz4x3gx3t9e0fh4hvqyu2qw4wvxm")?.assume_checked();
+    let receive_address1 =
+        Address::from_str("bcrt1qfjg5lv3dvc9az8patec8fjddrs4aqtauadnagr")?.assume_checked();
+
+    let misc_spks = [
+        receive_address0.script_pubkey(),
+        receive_address1.script_pubkey(),
+    ];
+
+    let _block_hashes = env.mine_blocks(101, None)?;
+    let txid1 = env.bitcoind.client.send_to_address(
+        &receive_address1,
+        Amount::from_sat(10000),
+        None,
+        None,
+        None,
+        None,
+        Some(1),
+        None,
+    )?;
+    let txid2 = env.bitcoind.client.send_to_address(
+        &receive_address0,
+        Amount::from_sat(20000),
+        None,
+        None,
+        None,
+        None,
+        Some(1),
+        None,
+    )?;
+    let _block_hashes = env.mine_blocks(1, None)?;
+    while env.client.get_height().await.unwrap() < 102 {
+        sleep(Duration::from_millis(10))
+    }
+
+    let graph_update = env
+        .client
+        .sync(
+            misc_spks.into_iter(),
+            vec![].into_iter(),
+            vec![].into_iter(),
+            1,
+        )
+        .await?;
+
+    // Check to see if we have the floating txouts available from our two created transactions'
+    // previous outputs in order to calculate transaction fees.
+    for tx in graph_update.full_txs() {
+        // Retrieve the calculated fee from `TxGraph`, which will panic if we do not have the
+        // floating txouts available from the transactions' previous outputs.
+        let fee = graph_update.calculate_fee(tx.tx).expect("Fee must exist");
+
+        // Retrieve the fee in the transaction data from `bitcoind`.
+        let tx_fee = env
+            .bitcoind
+            .client
+            .get_transaction(&tx.txid, None)
+            .expect("Tx must exist")
+            .fee
+            .expect("Fee must exist")
+            .abs()
+            .to_sat() as u64;
+
+        // Check that the calculated fee matches the fee from the transaction data.
+        assert_eq!(fee, tx_fee);
+    }
+
+    let mut graph_update_txids: Vec<Txid> = graph_update.full_txs().map(|tx| tx.txid).collect();
+    graph_update_txids.sort();
+    let mut expected_txids = vec![txid1, txid2];
+    expected_txids.sort();
+    assert_eq!(graph_update_txids, expected_txids);
+    Ok(())
+}
+
+/// Test the bounds of the address scan depending on the gap limit.
+#[tokio::test]
+pub async fn test_async_update_tx_graph_gap_limit() -> anyhow::Result<()> {
+    let env = TestEnv::new()?;
+    let _block_hashes = env.mine_blocks(101, None)?;
+
+    // Now let's test the gap limit. First of all get a chain of 10 addresses.
+    let addresses = [
+        "bcrt1qj9f7r8r3p2y0sqf4r3r62qysmkuh0fzep473d2ar7rcz64wqvhssjgf0z4",
+        "bcrt1qmm5t0ch7vh2hryx9ctq3mswexcugqe4atkpkl2tetm8merqkthas3w7q30",
+        "bcrt1qut9p7ej7l7lhyvekj28xknn8gnugtym4d5qvnp5shrsr4nksmfqsmyn87g",
+        "bcrt1qqz0xtn3m235p2k96f5wa2dqukg6shxn9n3txe8arlrhjh5p744hsd957ww",
+        "bcrt1q9c0t62a8l6wfytmf2t9lfj35avadk3mm8g4p3l84tp6rl66m48sqrme7wu",
+        "bcrt1qkmh8yrk2v47cklt8dytk8f3ammcwa4q7dzattedzfhqzvfwwgyzsg59zrh",
+        "bcrt1qvgrsrzy07gjkkfr5luplt0azxtfwmwq5t62gum5jr7zwcvep2acs8hhnp2",
+        "bcrt1qw57edarcg50ansq8mk3guyrk78rk0fwvrds5xvqeupteu848zayq549av8",
+        "bcrt1qvtve5ekf6e5kzs68knvnt2phfw6a0yjqrlgat392m6zt9jsvyxhqfx67ef",
+        "bcrt1qw03ddumfs9z0kcu76ln7jrjfdwam20qtffmkcral3qtza90sp9kqm787uk",
+    ];
+    let addresses: Vec<_> = addresses
+        .into_iter()
+        .map(|s| Address::from_str(s).unwrap().assume_checked())
+        .collect();
+    let spks: Vec<_> = addresses
+        .iter()
+        .enumerate()
+        .map(|(i, addr)| (i as u32, addr.script_pubkey()))
+        .collect();
+    let mut keychains = BTreeMap::new();
+    keychains.insert(0, spks);
+
+    // Then receive coins on the 4th address.
+    let txid_4th_addr = env.bitcoind.client.send_to_address(
+        &addresses[3],
+        Amount::from_sat(10000),
+        None,
+        None,
+        None,
+        None,
+        Some(1),
+        None,
+    )?;
+    let _block_hashes = env.mine_blocks(1, None)?;
+    while env.client.get_height().await.unwrap() < 103 {
+        sleep(Duration::from_millis(10))
+    }
+
+    // A scan with a gap limit of 2 won't find the transaction, but a scan with a gap limit of 3
+    // will.
+    let (graph_update, active_indices) = env.client.full_scan(keychains.clone(), 2, 1).await?;
+    assert!(graph_update.full_txs().next().is_none());
+    assert!(active_indices.is_empty());
+    let (graph_update, active_indices) = env.client.full_scan(keychains.clone(), 3, 1).await?;
+    assert_eq!(graph_update.full_txs().next().unwrap().txid, txid_4th_addr);
+    assert_eq!(active_indices[&0], 3);
+
+    // Now receive a coin on the last address.
+    let txid_last_addr = env.bitcoind.client.send_to_address(
+        &addresses[addresses.len() - 1],
+        Amount::from_sat(10000),
+        None,
+        None,
+        None,
+        None,
+        Some(1),
+        None,
+    )?;
+    let _block_hashes = env.mine_blocks(1, None)?;
+    while env.client.get_height().await.unwrap() < 104 {
+        sleep(Duration::from_millis(10))
+    }
+
+    // A scan with gap limit 4 won't find the second transaction, but a scan with gap limit 5 will.
+    // The last active indice won't be updated in the first case but will in the second one.
+    let (graph_update, active_indices) = env.client.full_scan(keychains.clone(), 4, 1).await?;
+    let txs: HashSet<_> = graph_update.full_txs().map(|tx| tx.txid).collect();
+    assert_eq!(txs.len(), 1);
+    assert!(txs.contains(&txid_4th_addr));
+    assert_eq!(active_indices[&0], 3);
+    let (graph_update, active_indices) = env.client.full_scan(keychains, 5, 1).await?;
+    let txs: HashSet<_> = graph_update.full_txs().map(|tx| tx.txid).collect();
+    assert_eq!(txs.len(), 2);
+    assert!(txs.contains(&txid_4th_addr) && txs.contains(&txid_last_addr));
+    assert_eq!(active_indices[&0], 9);
+
+    Ok(())
+}

crates/esplora/tests/blocking_ext.rs

@@ -0,0 +1,433 @@
+use bdk_chain::local_chain::LocalChain;
+use bdk_chain::BlockId;
+use bdk_esplora::EsploraExt;
+use electrsd::bitcoind::bitcoincore_rpc::RpcApi;
+use electrsd::bitcoind::{self, anyhow, BitcoinD};
+use electrsd::{Conf, ElectrsD};
+use esplora_client::{self, BlockingClient, Builder};
+use std::collections::{BTreeMap, BTreeSet, HashSet};
+use std::str::FromStr;
+use std::thread::sleep;
+use std::time::Duration;
+
+use bdk_chain::bitcoin::{Address, Amount, BlockHash, Txid};
+
+macro_rules! h {
+    ($index:literal) => {{
+        bdk_chain::bitcoin::hashes::Hash::hash($index.as_bytes())
+    }};
+}
+
+macro_rules! local_chain {
+    [ $(($height:expr, $block_hash:expr)), * ] => {{
+        #[allow(unused_mut)]
+        bdk_chain::local_chain::LocalChain::from_blocks([$(($height, $block_hash).into()),*].into_iter().collect())
+            .expect("chain must have genesis block")
+    }};
+}
+
+struct TestEnv {
+    bitcoind: BitcoinD,
+    #[allow(dead_code)]
+    electrsd: ElectrsD,
+    client: BlockingClient,
+}
+
+impl TestEnv {
+    fn new() -> Result<Self, anyhow::Error> {
+        let bitcoind_exe =
+            bitcoind::downloaded_exe_path().expect("bitcoind version feature must be enabled");
+        let bitcoind = BitcoinD::new(bitcoind_exe).unwrap();
+
+        let mut electrs_conf = Conf::default();
+        electrs_conf.http_enabled = true;
+        let electrs_exe =
+            electrsd::downloaded_exe_path().expect("electrs version feature must be enabled");
+        let electrsd = ElectrsD::with_conf(electrs_exe, &bitcoind, &electrs_conf)?;
+
+        let base_url = format!("http://{}", &electrsd.esplora_url.clone().unwrap());
+        let client = Builder::new(base_url.as_str()).build_blocking()?;
+
+        Ok(Self {
+            bitcoind,
+            electrsd,
+            client,
+        })
+    }
+
+    fn reset_electrsd(mut self) -> anyhow::Result<Self> {
+        let mut electrs_conf = Conf::default();
+        electrs_conf.http_enabled = true;
+        let electrs_exe =
+            electrsd::downloaded_exe_path().expect("electrs version feature must be enabled");
+        let electrsd = ElectrsD::with_conf(electrs_exe, &self.bitcoind, &electrs_conf)?;
+
+        let base_url = format!("http://{}", &electrsd.esplora_url.clone().unwrap());
+        let client = Builder::new(base_url.as_str()).build_blocking()?;
+        self.electrsd = electrsd;
+        self.client = client;
+        Ok(self)
+    }
+
+    fn mine_blocks(
+        &self,
+        count: usize,
+        address: Option<Address>,
+    ) -> anyhow::Result<Vec<BlockHash>> {
+        let coinbase_address = match address {
+            Some(address) => address,
+            None => self
+                .bitcoind
+                .client
+                .get_new_address(None, None)?
+                .assume_checked(),
+        };
+        let block_hashes = self
+            .bitcoind
+            .client
+            .generate_to_address(count as _, &coinbase_address)?;
+        Ok(block_hashes)
+    }
+}
+
+#[test]
+pub fn test_update_tx_graph_without_keychain() -> anyhow::Result<()> {
+    let env = TestEnv::new()?;
+    let receive_address0 =
+        Address::from_str("bcrt1qc6fweuf4xjvz4x3gx3t9e0fh4hvqyu2qw4wvxm")?.assume_checked();
+    let receive_address1 =
+        Address::from_str("bcrt1qfjg5lv3dvc9az8patec8fjddrs4aqtauadnagr")?.assume_checked();
+
+    let misc_spks = [
+        receive_address0.script_pubkey(),
+        receive_address1.script_pubkey(),
+    ];
+
+    let _block_hashes = env.mine_blocks(101, None)?;
+    let txid1 = env.bitcoind.client.send_to_address(
+        &receive_address1,
+        Amount::from_sat(10000),
+        None,
+        None,
+        None,
+        None,
+        Some(1),
+        None,
+    )?;
+    let txid2 = env.bitcoind.client.send_to_address(
+        &receive_address0,
+        Amount::from_sat(20000),
+        None,
+        None,
+        None,
+        None,
+        Some(1),
+        None,
+    )?;
+    let _block_hashes = env.mine_blocks(1, None)?;
+    while env.client.get_height().unwrap() < 102 {
+        sleep(Duration::from_millis(10))
+    }
+
+    let graph_update = env.client.sync(
+        misc_spks.into_iter(),
+        vec![].into_iter(),
+        vec![].into_iter(),
+        1,
+    )?;
+
+    // Check to see if we have the floating txouts available from our two created transactions'
+    // previous outputs in order to calculate transaction fees.
+    for tx in graph_update.full_txs() {
+        // Retrieve the calculated fee from `TxGraph`, which will panic if we do not have the
+        // floating txouts available from the transactions' previous outputs.
+        let fee = graph_update.calculate_fee(tx.tx).expect("Fee must exist");
+
+        // Retrieve the fee in the transaction data from `bitcoind`.
+        let tx_fee = env
+            .bitcoind
+            .client
+            .get_transaction(&tx.txid, None)
+            .expect("Tx must exist")
+            .fee
+            .expect("Fee must exist")
+            .abs()
+            .to_sat() as u64;
+
+        // Check that the calculated fee matches the fee from the transaction data.
+        assert_eq!(fee, tx_fee);
+    }
+
+    let mut graph_update_txids: Vec<Txid> = graph_update.full_txs().map(|tx| tx.txid).collect();
+    graph_update_txids.sort();
+    let mut expected_txids = vec![txid1, txid2];
+    expected_txids.sort();
+    assert_eq!(graph_update_txids, expected_txids);
+
+    Ok(())
+}
+
+/// Test the bounds of the address scan depending on the gap limit.
+#[test]
+pub fn test_update_tx_graph_gap_limit() -> anyhow::Result<()> {
+    let env = TestEnv::new()?;
+    let _block_hashes = env.mine_blocks(101, None)?;
+
+    // Now let's test the gap limit. First of all get a chain of 10 addresses.
+    let addresses = [
+        "bcrt1qj9f7r8r3p2y0sqf4r3r62qysmkuh0fzep473d2ar7rcz64wqvhssjgf0z4",
+        "bcrt1qmm5t0ch7vh2hryx9ctq3mswexcugqe4atkpkl2tetm8merqkthas3w7q30",
+        "bcrt1qut9p7ej7l7lhyvekj28xknn8gnugtym4d5qvnp5shrsr4nksmfqsmyn87g",
+        "bcrt1qqz0xtn3m235p2k96f5wa2dqukg6shxn9n3txe8arlrhjh5p744hsd957ww",
+        "bcrt1q9c0t62a8l6wfytmf2t9lfj35avadk3mm8g4p3l84tp6rl66m48sqrme7wu",
+        "bcrt1qkmh8yrk2v47cklt8dytk8f3ammcwa4q7dzattedzfhqzvfwwgyzsg59zrh",
+        "bcrt1qvgrsrzy07gjkkfr5luplt0azxtfwmwq5t62gum5jr7zwcvep2acs8hhnp2",
+        "bcrt1qw57edarcg50ansq8mk3guyrk78rk0fwvrds5xvqeupteu848zayq549av8",
+        "bcrt1qvtve5ekf6e5kzs68knvnt2phfw6a0yjqrlgat392m6zt9jsvyxhqfx67ef",
+        "bcrt1qw03ddumfs9z0kcu76ln7jrjfdwam20qtffmkcral3qtza90sp9kqm787uk",
+    ];
+    let addresses: Vec<_> = addresses
+        .into_iter()
+        .map(|s| Address::from_str(s).unwrap().assume_checked())
+        .collect();
+    let spks: Vec<_> = addresses
+        .iter()
+        .enumerate()
+        .map(|(i, addr)| (i as u32, addr.script_pubkey()))
+        .collect();
+    let mut keychains = BTreeMap::new();
+    keychains.insert(0, spks);
+
+    // Then receive coins on the 4th address.
+    let txid_4th_addr = env.bitcoind.client.send_to_address(
+        &addresses[3],
+        Amount::from_sat(10000),
+        None,
+        None,
+        None,
+        None,
+        Some(1),
+        None,
+    )?;
+    let _block_hashes = env.mine_blocks(1, None)?;
+    while env.client.get_height().unwrap() < 103 {
+        sleep(Duration::from_millis(10))
+    }
+
+    // A scan with a gap limit of 2 won't find the transaction, but a scan with a gap limit of 3
+    // will.
+    let (graph_update, active_indices) = env.client.full_scan(keychains.clone(), 2, 1)?;
+    assert!(graph_update.full_txs().next().is_none());
+    assert!(active_indices.is_empty());
+    let (graph_update, active_indices) = env.client.full_scan(keychains.clone(), 3, 1)?;
+    assert_eq!(graph_update.full_txs().next().unwrap().txid, txid_4th_addr);
+    assert_eq!(active_indices[&0], 3);
+
+    // Now receive a coin on the last address.
+    let txid_last_addr = env.bitcoind.client.send_to_address(
+        &addresses[addresses.len() - 1],
+        Amount::from_sat(10000),
+        None,
+        None,
+        None,
+        None,
+        Some(1),
+        None,
+    )?;
+    let _block_hashes = env.mine_blocks(1, None)?;
+    while env.client.get_height().unwrap() < 104 {
+        sleep(Duration::from_millis(10))
+    }
+
+    // A scan with gap limit 4 won't find the second transaction, but a scan with gap limit 5 will.
+    // The last active indice won't be updated in the first case but will in the second one.
+    let (graph_update, active_indices) = env.client.full_scan(keychains.clone(), 4, 1)?;
+    let txs: HashSet<_> = graph_update.full_txs().map(|tx| tx.txid).collect();
+    assert_eq!(txs.len(), 1);
+    assert!(txs.contains(&txid_4th_addr));
+    assert_eq!(active_indices[&0], 3);
+    let (graph_update, active_indices) = env.client.full_scan(keychains, 5, 1)?;
+    let txs: HashSet<_> = graph_update.full_txs().map(|tx| tx.txid).collect();
+    assert_eq!(txs.len(), 2);
+    assert!(txs.contains(&txid_4th_addr) && txs.contains(&txid_last_addr));
+    assert_eq!(active_indices[&0], 9);
+
+    Ok(())
+}
+
+#[test]
+fn update_local_chain() -> anyhow::Result<()> {
+    const TIP_HEIGHT: u32 = 50;
+
+    let env = TestEnv::new()?;
+    let blocks = {
+        let bitcoind_client = &env.bitcoind.client;
+        assert_eq!(bitcoind_client.get_block_count()?, 1);
+        [
+            (0, bitcoind_client.get_block_hash(0)?),
+            (1, bitcoind_client.get_block_hash(1)?),
+        ]
+        .into_iter()
+        .chain((2..).zip(env.mine_blocks((TIP_HEIGHT - 1) as usize, None)?))
+        .collect::<BTreeMap<_, _>>()
+    };
+    // so new blocks can be seen by Electrs
+    let env = env.reset_electrsd()?;
+
+    struct TestCase {
+        name: &'static str,
+        chain: LocalChain,
+        request_heights: &'static [u32],
+        exp_update_heights: &'static [u32],
+    }
+
+    let test_cases = [
+        TestCase {
+            name: "request_later_blocks",
+            chain: local_chain![(0, blocks[&0]), (21, blocks[&21])],
+            request_heights: &[22, 25, 28],
+            exp_update_heights: &[21, 22, 25, 28],
+        },
+        TestCase {
+            name: "request_prev_blocks",
+            chain: local_chain![(0, blocks[&0]), (1, blocks[&1]), (5, blocks[&5])],
+            request_heights: &[4],
+            exp_update_heights: &[4, 5],
+        },
+        TestCase {
+            name: "request_prev_blocks_2",
+            chain: local_chain![(0, blocks[&0]), (1, blocks[&1]), (10, blocks[&10])],
+            request_heights: &[4, 6],
+            exp_update_heights: &[4, 6, 10],
+        },
+        TestCase {
+            name: "request_later_and_prev_blocks",
+            chain: local_chain![(0, blocks[&0]), (7, blocks[&7]), (11, blocks[&11])],
+            request_heights: &[8, 9, 15],
+            exp_update_heights: &[8, 9, 11, 15],
+        },
+        TestCase {
+            name: "request_tip_only",
+            chain: local_chain![(0, blocks[&0]), (5, blocks[&5]), (49, blocks[&49])],
+            request_heights: &[TIP_HEIGHT],
+            exp_update_heights: &[49],
+        },
+        TestCase {
+            name: "request_nothing",
+            chain: local_chain![(0, blocks[&0]), (13, blocks[&13]), (23, blocks[&23])],
+            request_heights: &[],
+            exp_update_heights: &[23],
+        },
+        TestCase {
+            name: "request_nothing_during_reorg",
+            chain: local_chain![(0, blocks[&0]), (13, blocks[&13]), (23, h!("23"))],
+            request_heights: &[],
+            exp_update_heights: &[13, 23],
+        },
+        TestCase {
+            name: "request_nothing_during_reorg_2",
+            chain: local_chain![
+                (0, blocks[&0]),
+                (21, blocks[&21]),
+                (22, h!("22")),
+                (23, h!("23"))
+            ],
+            request_heights: &[],
+            exp_update_heights: &[21, 22, 23],
+        },
+        TestCase {
+            name: "request_prev_blocks_during_reorg",
+            chain: local_chain![
+                (0, blocks[&0]),
+                (21, blocks[&21]),
+                (22, h!("22")),
+                (23, h!("23"))
+            ],
+            request_heights: &[17, 20],
+            exp_update_heights: &[17, 20, 21, 22, 23],
+        },
+        TestCase {
+            name: "request_later_blocks_during_reorg",
+            chain: local_chain![
+                (0, blocks[&0]),
+                (9, blocks[&9]),
+                (22, h!("22")),
+                (23, h!("23"))
+            ],
+            request_heights: &[25, 27],
+            exp_update_heights: &[9, 22, 23, 25, 27],
+        },
+        TestCase {
+            name: "request_later_blocks_during_reorg_2",
+            chain: local_chain![(0, blocks[&0]), (9, h!("9"))],
+            request_heights: &[10],
+            exp_update_heights: &[0, 9, 10],
+        },
+        TestCase {
+            name: "request_later_and_prev_blocks_during_reorg",
+            chain: local_chain![(0, blocks[&0]), (1, blocks[&1]), (9, h!("9"))],
+            request_heights: &[8, 11],
+            exp_update_heights: &[1, 8, 9, 11],
+        },
+    ];
+
+    for (i, t) in test_cases.into_iter().enumerate() {
+        println!("Case {}: {}", i, t.name);
+        let mut chain = t.chain;
+
+        let update = env
+            .client
+            .update_local_chain(chain.tip(), t.request_heights.iter().copied())
+            .map_err(|err| {
+                anyhow::format_err!("[{}:{}] `update_local_chain` failed: {}", i, t.name, err)
+            })?;
+
+        let update_blocks = update
+            .tip
+            .iter()
+            .map(|cp| cp.block_id())
+            .collect::<BTreeSet<_>>();
+
+        let exp_update_blocks = t
+            .exp_update_heights
+            .iter()
+            .map(|&height| {
+                let hash = blocks[&height];
+                BlockId { height, hash }
+            })
+            .chain(
+                // Electrs Esplora `get_block` call fetches 10 blocks which is included in the
+                // update
+                blocks
+                    .range(TIP_HEIGHT - 9..)
+                    .map(|(&height, &hash)| BlockId { height, hash }),
+            )
+            .collect::<BTreeSet<_>>();
+
+        assert_eq!(
+            update_blocks, exp_update_blocks,
+            "[{}:{}] unexpected update",
+            i, t.name
+        );
+
+        let _ = chain
+            .apply_update(update)
+            .unwrap_or_else(|err| panic!("[{}:{}] update failed to apply: {}", i, t.name, err));
+
+        // all requested heights must exist in the final chain
+        for height in t.request_heights {
+            let exp_blockhash = blocks.get(height).expect("block must exist in bitcoind");
+            assert_eq!(
+                chain.blocks().get(height),
+                Some(exp_blockhash),
+                "[{}:{}] block {}:{} must exist in final chain",
+                i,
+                t.name,
+                height,
+                exp_blockhash
+            );
+        }
+    }
+
+    Ok(())
+}

crates/file_store/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "bdk_file_store"
-version = "0.2.0"
+version = "0.7.0"
 edition = "2021"
 license = "MIT OR Apache-2.0"
 repository = "https://github.com/bitcoindevkit/bdk"
@@ -11,7 +11,7 @@ authors = ["Bitcoin Dev Kit Developers"]
 readme = "README.md"
 
 [dependencies]
-bdk_chain = { path = "../chain", version = "0.5.0", features = [ "serde", "miniscript" ] }
+bdk_chain = { path = "../chain", version = "0.11.0", features = [ "serde", "miniscript" ] }
 bincode = { version = "1" }
 serde = { version = "1", features = ["derive"] }
 

crates/file_store/src/entry_iter.rs

@@ -1,7 +1,7 @@
 use bincode::Options;
 use std::{
     fs::File,
-    io::{self, Seek},
+    io::{self, BufReader, Seek},
     marker::PhantomData,
 };
 
@@ -14,8 +14,9 @@ use crate::bincode_options;
 ///
 /// [`next`]: Self::next
 pub struct EntryIter<'t, T> {
-    db_file: Option<&'t mut File>,
-
+    /// Buffered reader around the file
+    db_file: BufReader<&'t mut File>,
+    finished: bool,
     /// The file position for the first read of `db_file`.
     start_pos: Option<u64>,
     types: PhantomData<T>,
@@ -24,8 +25,9 @@ pub struct EntryIter<'t, T> {
 impl<'t, T> EntryIter<'t, T> {
     pub fn new(start_pos: u64, db_file: &'t mut File) -> Self {
         Self {
-            db_file: Some(db_file),
+            db_file: BufReader::new(db_file),
             start_pos: Some(start_pos),
+            finished: false,
             types: PhantomData,
         }
     }
@@ -38,44 +40,44 @@ where
     type Item = Result<T, IterError>;
 
     fn next(&mut self) -> Option<Self::Item> {
-        // closure which reads a single entry starting from `self.pos`
-        let read_one = |f: &mut File, start_pos: Option<u64>| -> Result<Option<T>, IterError> {
-            let pos = match start_pos {
-                Some(pos) => f.seek(io::SeekFrom::Start(pos))?,
-                None => f.stream_position()?,
-            };
+        if self.finished {
+            return None;
+        }
+        (|| {
+            if let Some(start) = self.start_pos.take() {
+                self.db_file.seek(io::SeekFrom::Start(start))?;
+            }
 
-            match bincode_options().deserialize_from(&*f) {
-                Ok(changeset) => {
-                    f.stream_position()?;
-                    Ok(Some(changeset))
-                }
+            let pos_before_read = self.db_file.stream_position()?;
+            match bincode_options().deserialize_from(&mut self.db_file) {
+                Ok(changeset) => Ok(Some(changeset)),
                 Err(e) => {
+                    self.finished = true;
+                    let pos_after_read = self.db_file.stream_position()?;
+                    // allow unexpected EOF if 0 bytes were read
                     if let bincode::ErrorKind::Io(inner) = &*e {
-                        if inner.kind() == io::ErrorKind::UnexpectedEof {
-                            let eof = f.seek(io::SeekFrom::End(0))?;
-                            if pos == eof {
-                                return Ok(None);
-                            }
+                        if inner.kind() == io::ErrorKind::UnexpectedEof
+                            && pos_after_read == pos_before_read
+                        {
+                            return Ok(None);
                         }
                     }
-                    f.seek(io::SeekFrom::Start(pos))?;
+                    self.db_file.seek(io::SeekFrom::Start(pos_before_read))?;
                     Err(IterError::Bincode(*e))
                 }
             }
-        };
-
-        let result = read_one(self.db_file.as_mut()?, self.start_pos.take());
-        if result.is_err() {
-            self.db_file = None;
-        }
-        result.transpose()
+        })()
+        .transpose()
     }
 }
 
-impl From<io::Error> for IterError {
-    fn from(value: io::Error) -> Self {
-        IterError::Io(value)
+impl<'t, T> Drop for EntryIter<'t, T> {
+    fn drop(&mut self) {
+        // This syncs the underlying file's offset with the buffer's position. This way, we
+        // maintain the correct position to start the next read/write.
+        if let Ok(pos) = self.db_file.stream_position() {
+            let _ = self.db_file.get_mut().seek(io::SeekFrom::Start(pos));
+        }
     }
 }
 
@@ -97,4 +99,10 @@ impl core::fmt::Display for IterError {
     }
 }
 
+impl From<io::Error> for IterError {
+    fn from(value: io::Error) -> Self {
+        IterError::Io(value)
+    }
+}
+
 impl std::error::Error for IterError {}

crates/file_store/src/lib.rs

@@ -13,14 +13,14 @@ pub(crate) fn bincode_options() -> impl bincode::Options {
 
 /// Error that occurs due to problems encountered with the file.
 #[derive(Debug)]
-pub enum FileError<'a> {
+pub enum FileError {
     /// IO error, this may mean that the file is too short.
     Io(io::Error),
     /// Magic bytes do not match what is expected.
-    InvalidMagicBytes { got: Vec<u8>, expected: &'a [u8] },
+    InvalidMagicBytes { got: Vec<u8>, expected: Vec<u8> },
 }
 
-impl<'a> core::fmt::Display for FileError<'a> {
+impl core::fmt::Display for FileError {
     fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
         match self {
             Self::Io(e) => write!(f, "io error trying to read file: {}", e),
@@ -33,10 +33,10 @@ impl<'a> core::fmt::Display for FileError<'a> {
     }
 }
 
-impl<'a> From<io::Error> for FileError<'a> {
+impl From<io::Error> for FileError {
     fn from(value: io::Error) -> Self {
         Self::Io(value)
     }
 }
 
-impl<'a> std::error::Error for FileError<'a> {}
+impl std::error::Error for FileError {}

crates/file_store/src/store.rs

@@ -15,15 +15,15 @@ use crate::{bincode_options, EntryIter, FileError, IterError};
 ///
 /// The changesets are the results of altering a tracker implementation (`T`).
 #[derive(Debug)]
-pub struct Store<'a, C> {
-    magic: &'a [u8],
+pub struct Store<C> {
+    magic_len: usize,
     db_file: File,
     marker: PhantomData<C>,
 }
 
-impl<'a, C> PersistBackend<C> for Store<'a, C>
+impl<C> PersistBackend<C> for Store<C>
 where
-    C: Default + Append + serde::Serialize + serde::de::DeserializeOwned,
+    C: Append + serde::Serialize + serde::de::DeserializeOwned,
 {
     type WriteError = std::io::Error;
 
@@ -33,68 +33,93 @@ where
         self.append_changeset(changeset)
     }
 
-    fn load_from_persistence(&mut self) -> Result<C, Self::LoadError> {
-        let (changeset, result) = self.aggregate_changesets();
-        result.map(|_| changeset)
+    fn load_from_persistence(&mut self) -> Result<Option<C>, Self::LoadError> {
+        self.aggregate_changesets().map_err(|e| e.iter_error)
     }
 }
 
-impl<'a, C> Store<'a, C>
+impl<C> Store<C>
 where
-    C: Default + Append + serde::Serialize + serde::de::DeserializeOwned,
+    C: Append + serde::Serialize + serde::de::DeserializeOwned,
 {
-    /// Creates a new store from a [`File`].
+    /// Create a new [`Store`] file in write-only mode; error if the file exists.
     ///
-    /// The file must have been opened with read and write permissions.
+    /// `magic` is the prefixed bytes to write to the new file. This will be checked when opening
+    /// the `Store` in the future with [`open`].
     ///
-    /// `magic` is the expected prefixed bytes of the file. If this does not match, an error will be
-    /// returned.
-    ///
-    /// [`File`]: std::fs::File
-    pub fn new(magic: &'a [u8], mut db_file: File) -> Result<Self, FileError> {
-        db_file.rewind()?;
-
-        let mut magic_buf = vec![0_u8; magic.len()];
-        db_file.read_exact(magic_buf.as_mut())?;
-
-        if magic_buf != magic {
-            return Err(FileError::InvalidMagicBytes {
-                got: magic_buf,
-                expected: magic,
-            });
+    /// [`open`]: Store::open
+    pub fn create_new<P>(magic: &[u8], file_path: P) -> Result<Self, FileError>
+    where
+        P: AsRef<Path>,
+    {
+        if file_path.as_ref().exists() {
+            // `io::Error` is used instead of a variant on `FileError` because there is already a
+            // nightly-only `File::create_new` method
+            return Err(FileError::Io(io::Error::new(
+                io::ErrorKind::Other,
+                "file already exists",
+            )));
         }
-
+        let mut f = OpenOptions::new()
+            .create(true)
+            .read(true)
+            .write(true)
+            .open(file_path)?;
+        f.write_all(magic)?;
         Ok(Self {
-            magic,
-            db_file,
+            magic_len: magic.len(),
+            db_file: f,
             marker: Default::default(),
         })
     }
 
-    /// Creates or loads a store from `db_path`.
+    /// Open an existing [`Store`].
     ///
-    /// If no file exists there, it will be created.
+    /// Use [`create_new`] to create a new `Store`.
     ///
-    /// Refer to [`new`] for documentation on the `magic` input.
+    /// # Errors
     ///
-    /// [`new`]: Self::new
-    pub fn new_from_path<P>(magic: &'a [u8], db_path: P) -> Result<Self, FileError>
+    /// If the prefixed bytes of the opened file does not match the provided `magic`, the
+    /// [`FileError::InvalidMagicBytes`] error variant will be returned.
+    ///
+    /// [`create_new`]: Store::create_new
+    pub fn open<P>(magic: &[u8], file_path: P) -> Result<Self, FileError>
     where
         P: AsRef<Path>,
     {
-        let already_exists = db_path.as_ref().exists();
+        let mut f = OpenOptions::new().read(true).write(true).open(file_path)?;
 
-        let mut db_file = OpenOptions::new()
-            .read(true)
-            .write(true)
-            .create(true)
-            .open(db_path)?;
-
-        if !already_exists {
-            db_file.write_all(magic)?;
+        let mut magic_buf = vec![0_u8; magic.len()];
+        f.read_exact(&mut magic_buf)?;
+        if magic_buf != magic {
+            return Err(FileError::InvalidMagicBytes {
+                got: magic_buf,
+                expected: magic.to_vec(),
+            });
         }
 
-        Self::new(magic, db_file)
+        Ok(Self {
+            magic_len: magic.len(),
+            db_file: f,
+            marker: Default::default(),
+        })
+    }
+
+    /// Attempt to open existing [`Store`] file; create it if the file is non-existent.
+    ///
+    /// Internally, this calls either [`open`] or [`create_new`].
+    ///
+    /// [`open`]: Store::open
+    /// [`create_new`]: Store::create_new
+    pub fn open_or_create_new<P>(magic: &[u8], file_path: P) -> Result<Self, FileError>
+    where
+        P: AsRef<Path>,
+    {
+        if file_path.as_ref().exists() {
+            Self::open(magic, file_path)
+        } else {
+            Self::create_new(magic, file_path)
+        }
     }
 
     /// Iterates over the stored changeset from first to last, changing the seek position at each
@@ -107,14 +132,14 @@ where
     /// always iterate over all entries until `None` is returned if you want your next write to go
     /// at the end; otherwise, you will write over existing entries.
     pub fn iter_changesets(&mut self) -> EntryIter<C> {
-        EntryIter::new(self.magic.len() as u64, &mut self.db_file)
+        EntryIter::new(self.magic_len as u64, &mut self.db_file)
     }
 
     /// Loads all the changesets that have been stored as one giant changeset.
     ///
-    /// This function returns a tuple of the aggregate changeset and a result that indicates
-    /// whether an error occurred while reading or deserializing one of the entries. If so the
-    /// changeset will consist of all of those it was able to read.
+    /// This function returns the aggregate changeset, or `None` if nothing was persisted.
+    /// If reading or deserializing any of the entries fails, an error is returned that
+    /// consists of all those it was able to read.
     ///
     /// You should usually check the error. In many applications, it may make sense to do a full
     /// wallet scan with a stop-gap after getting an error, since it is likely that one of the
@@ -122,16 +147,24 @@ where
     ///
     /// **WARNING**: This method changes the write position of the underlying file. The next
     /// changeset will be written over the erroring entry (or the end of the file if none existed).
-    pub fn aggregate_changesets(&mut self) -> (C, Result<(), IterError>) {
-        let mut changeset = C::default();
-        let result = (|| {
-            for next_changeset in self.iter_changesets() {
-                changeset.append(next_changeset?);
+    pub fn aggregate_changesets(&mut self) -> Result<Option<C>, AggregateChangesetsError<C>> {
+        let mut changeset = Option::<C>::None;
+        for next_changeset in self.iter_changesets() {
+            let next_changeset = match next_changeset {
+                Ok(next_changeset) => next_changeset,
+                Err(iter_error) => {
+                    return Err(AggregateChangesetsError {
+                        changeset,
+                        iter_error,
+                    })
+                }
+            };
+            match &mut changeset {
+                Some(changeset) => changeset.append(next_changeset),
+                changeset => *changeset = Some(next_changeset),
             }
-            Ok(())
-        })();
-
-        (changeset, result)
+        }
+        Ok(changeset)
     }
 
     /// Append a new changeset to the file and truncate the file to the end of the appended
@@ -162,12 +195,31 @@ where
     }
 }
 
+/// Error type for [`Store::aggregate_changesets`].
+#[derive(Debug)]
+pub struct AggregateChangesetsError<C> {
+    /// The partially-aggregated changeset.
+    pub changeset: Option<C>,
+
+    /// The error returned by [`EntryIter`].
+    pub iter_error: IterError,
+}
+
+impl<C> std::fmt::Display for AggregateChangesetsError<C> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        std::fmt::Display::fmt(&self.iter_error, f)
+    }
+}
+
+impl<C: std::fmt::Debug> std::error::Error for AggregateChangesetsError<C> {}
+
 #[cfg(test)]
 mod test {
     use super::*;
 
     use bincode::DefaultOptions;
     use std::{
+        collections::BTreeSet,
         io::{Read, Write},
         vec::Vec,
     };
@@ -177,18 +229,55 @@ mod test {
     const TEST_MAGIC_BYTES: [u8; TEST_MAGIC_BYTES_LEN] =
         [98, 100, 107, 102, 115, 49, 49, 49, 49, 49, 49, 49];
 
-    type TestChangeSet = Vec<String>;
+    type TestChangeSet = BTreeSet<String>;
 
     #[derive(Debug)]
     struct TestTracker;
 
+    /// Check behavior of [`Store::create_new`] and [`Store::open`].
+    #[test]
+    fn construct_store() {
+        let temp_dir = tempfile::tempdir().unwrap();
+        let file_path = temp_dir.path().join("db_file");
+        let _ = Store::<TestChangeSet>::open(&TEST_MAGIC_BYTES, &file_path)
+            .expect_err("must not open as file does not exist yet");
+        let _ = Store::<TestChangeSet>::create_new(&TEST_MAGIC_BYTES, &file_path)
+            .expect("must create file");
+        // cannot create new as file already exists
+        let _ = Store::<TestChangeSet>::create_new(&TEST_MAGIC_BYTES, &file_path)
+            .expect_err("must fail as file already exists now");
+        let _ = Store::<TestChangeSet>::open(&TEST_MAGIC_BYTES, &file_path)
+            .expect("must open as file exists now");
+    }
+
+    #[test]
+    fn open_or_create_new() {
+        let temp_dir = tempfile::tempdir().unwrap();
+        let file_path = temp_dir.path().join("db_file");
+        let changeset = BTreeSet::from(["hello".to_string(), "world".to_string()]);
+
+        {
+            let mut db = Store::<TestChangeSet>::open_or_create_new(&TEST_MAGIC_BYTES, &file_path)
+                .expect("must create");
+            assert!(file_path.exists());
+            db.append_changeset(&changeset).expect("must succeed");
+        }
+
+        {
+            let mut db = Store::<TestChangeSet>::open_or_create_new(&TEST_MAGIC_BYTES, &file_path)
+                .expect("must recover");
+            let recovered_changeset = db.aggregate_changesets().expect("must succeed");
+            assert_eq!(recovered_changeset, Some(changeset));
+        }
+    }
+
     #[test]
     fn new_fails_if_file_is_too_short() {
         let mut file = NamedTempFile::new().unwrap();
         file.write_all(&TEST_MAGIC_BYTES[..TEST_MAGIC_BYTES_LEN - 1])
             .expect("should write");
 
-        match Store::<TestChangeSet>::new(&TEST_MAGIC_BYTES, file.reopen().unwrap()) {
+        match Store::<TestChangeSet>::open(&TEST_MAGIC_BYTES, file.path()) {
             Err(FileError::Io(e)) => assert_eq!(e.kind(), std::io::ErrorKind::UnexpectedEof),
             unexpected => panic!("unexpected result: {:?}", unexpected),
         };
@@ -202,7 +291,7 @@ mod test {
         file.write_all(invalid_magic_bytes.as_bytes())
             .expect("should write");
 
-        match Store::<TestChangeSet>::new(&TEST_MAGIC_BYTES, file.reopen().unwrap()) {
+        match Store::<TestChangeSet>::open(&TEST_MAGIC_BYTES, file.path()) {
             Err(FileError::InvalidMagicBytes { got, .. }) => {
                 assert_eq!(got, invalid_magic_bytes.as_bytes())
             }
@@ -216,13 +305,13 @@ mod test {
         let mut data = [255_u8; 2000];
         data[..TEST_MAGIC_BYTES_LEN].copy_from_slice(&TEST_MAGIC_BYTES);
 
-        let changeset = vec!["one".into(), "two".into(), "three!".into()];
+        let changeset = TestChangeSet::from(["one".into(), "two".into(), "three!".into()]);
 
         let mut file = NamedTempFile::new().unwrap();
         file.write_all(&data).expect("should write");
 
-        let mut store = Store::<TestChangeSet>::new(&TEST_MAGIC_BYTES, file.reopen().unwrap())
-            .expect("should open");
+        let mut store =
+            Store::<TestChangeSet>::open(&TEST_MAGIC_BYTES, file.path()).expect("should open");
         match store.iter_changesets().next() {
             Some(Err(IterError::Bincode(_))) => {}
             unexpected_res => panic!("unexpected result: {:?}", unexpected_res),
@@ -252,4 +341,119 @@ mod test {
 
         assert_eq!(got_bytes, expected_bytes);
     }
+
+    #[test]
+    fn last_write_is_short() {
+        let temp_dir = tempfile::tempdir().unwrap();
+
+        let changesets = [
+            TestChangeSet::from(["1".into()]),
+            TestChangeSet::from(["2".into(), "3".into()]),
+            TestChangeSet::from(["4".into(), "5".into(), "6".into()]),
+        ];
+        let last_changeset = TestChangeSet::from(["7".into(), "8".into(), "9".into()]);
+        let last_changeset_bytes = bincode_options().serialize(&last_changeset).unwrap();
+
+        for short_write_len in 1..last_changeset_bytes.len() - 1 {
+            let file_path = temp_dir.path().join(format!("{}.dat", short_write_len));
+            println!("Test file: {:?}", file_path);
+
+            // simulate creating a file, writing data where the last write is incomplete
+            {
+                let mut db =
+                    Store::<TestChangeSet>::create_new(&TEST_MAGIC_BYTES, &file_path).unwrap();
+                for changeset in &changesets {
+                    db.append_changeset(changeset).unwrap();
+                }
+                // this is the incomplete write
+                db.db_file
+                    .write_all(&last_changeset_bytes[..short_write_len])
+                    .unwrap();
+            }
+
+            // load file again and aggregate changesets
+            // write the last changeset again (this time it succeeds)
+            {
+                let mut db = Store::<TestChangeSet>::open(&TEST_MAGIC_BYTES, &file_path).unwrap();
+                let err = db
+                    .aggregate_changesets()
+                    .expect_err("should return error as last read is short");
+                assert_eq!(
+                    err.changeset,
+                    changesets.iter().cloned().reduce(|mut acc, cs| {
+                        Append::append(&mut acc, cs);
+                        acc
+                    }),
+                    "should recover all changesets that are written in full",
+                );
+                db.db_file.write_all(&last_changeset_bytes).unwrap();
+            }
+
+            // load file again - this time we should successfully aggregate all changesets
+            {
+                let mut db = Store::<TestChangeSet>::open(&TEST_MAGIC_BYTES, &file_path).unwrap();
+                let aggregated_changesets = db
+                    .aggregate_changesets()
+                    .expect("aggregating all changesets should succeed");
+                assert_eq!(
+                    aggregated_changesets,
+                    changesets
+                        .iter()
+                        .cloned()
+                        .chain(core::iter::once(last_changeset.clone()))
+                        .reduce(|mut acc, cs| {
+                            Append::append(&mut acc, cs);
+                            acc
+                        }),
+                    "should recover all changesets",
+                );
+            }
+        }
+    }
+
+    #[test]
+    fn write_after_short_read() {
+        let temp_dir = tempfile::tempdir().unwrap();
+
+        let changesets = (0..20)
+            .map(|n| TestChangeSet::from([format!("{}", n)]))
+            .collect::<Vec<_>>();
+        let last_changeset = TestChangeSet::from(["last".into()]);
+
+        for read_count in 0..changesets.len() {
+            let file_path = temp_dir.path().join(format!("{}.dat", read_count));
+            println!("Test file: {:?}", file_path);
+
+            // First, we create the file with all the changesets!
+            let mut db = Store::<TestChangeSet>::create_new(&TEST_MAGIC_BYTES, &file_path).unwrap();
+            for changeset in &changesets {
+                db.append_changeset(changeset).unwrap();
+            }
+            drop(db);
+
+            // We re-open the file and read `read_count` number of changesets.
+            let mut db = Store::<TestChangeSet>::open(&TEST_MAGIC_BYTES, &file_path).unwrap();
+            let mut exp_aggregation = db
+                .iter_changesets()
+                .take(read_count)
+                .map(|r| r.expect("must read valid changeset"))
+                .fold(TestChangeSet::default(), |mut acc, v| {
+                    Append::append(&mut acc, v);
+                    acc
+                });
+            // We write after a short read.
+            db.write_changes(&last_changeset)
+                .expect("last write must succeed");
+            Append::append(&mut exp_aggregation, last_changeset.clone());
+            drop(db);
+
+            // We open the file again and check whether aggregate changeset is expected.
+            let aggregation = Store::<TestChangeSet>::open(&TEST_MAGIC_BYTES, &file_path)
+                .unwrap()
+                .aggregate_changesets()
+                .expect("must aggregate changesets")
+                .unwrap_or_default();
+            assert_eq!(aggregation, exp_aggregation);
+        }
+    }
 }

crates/file_store/tests/test_file_store.rs

@@ -1 +0,0 @@
-

crates/hwi/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "bdk_hwi"
+version = "0.1.0"
+edition = "2021"
+homepage = "https://bitcoindevkit.org"
+repository = "https://github.com/bitcoindevkit/bdk"
+description = "Utilities to use bdk with hardware wallets"
+license = "MIT OR Apache-2.0"
+readme = "README.md"
+
+[dependencies]
+bdk = { path = "../bdk" }
+hwi = { version = "0.7.0", features = [ "miniscript"] }

crates/hwi/src/lib.rs

@@ -0,0 +1,42 @@
+//! HWI Signer
+//!
+//! This crate contains HWISigner, an implementation of a [`TransactionSigner`] to be
+//! used with hardware wallets.
+//! ```no_run
+//! # use bdk::bitcoin::Network;
+//! # use bdk::signer::SignerOrdering;
+//! # use bdk_hwi::HWISigner;
+//! # use bdk::wallet::AddressIndex::New;
+//! # use bdk::{FeeRate, KeychainKind, SignOptions, Wallet};
+//! # use hwi::HWIClient;
+//! # use std::sync::Arc;
+//! #
+//! # fn main() -> Result<(), Box<dyn std::error::Error>> {
+//! let mut devices = HWIClient::enumerate()?;
+//! if devices.is_empty() {
+//!     panic!("No devices found!");
+//! }
+//! let first_device = devices.remove(0)?;
+//! let custom_signer = HWISigner::from_device(&first_device, Network::Testnet.into())?;
+//!
+//! # let mut wallet = Wallet::new_no_persist(
+//! #     "",
+//! #     None,
+//! #     Network::Testnet,
+//! # )?;
+//! #
+//! // Adding the hardware signer to the BDK wallet
+//! wallet.add_signer(
+//!     KeychainKind::External,
+//!     SignerOrdering(200),
+//!     Arc::new(custom_signer),
+//! );
+//!
+//! # Ok(())
+//! # }
+//! ```
+//!
+//! [`TransactionSigner`]: bdk::wallet::signer::TransactionSigner
+
+mod signer;
+pub use signer::*;

crates/hwi/src/signer.rs

@@ -0,0 +1,94 @@
+use bdk::bitcoin::bip32::Fingerprint;
+use bdk::bitcoin::psbt::PartiallySignedTransaction;
+use bdk::bitcoin::secp256k1::{All, Secp256k1};
+
+use hwi::error::Error;
+use hwi::types::{HWIChain, HWIDevice};
+use hwi::HWIClient;
+
+use bdk::signer::{SignerCommon, SignerError, SignerId, TransactionSigner};
+
+#[derive(Debug)]
+/// Custom signer for Hardware Wallets
+///
+/// This ignores `sign_options` and leaves the decisions up to the hardware wallet.
+pub struct HWISigner {
+    fingerprint: Fingerprint,
+    client: HWIClient,
+}
+
+impl HWISigner {
+    /// Create a instance from the specified device and chain
+    pub fn from_device(device: &HWIDevice, chain: HWIChain) -> Result<HWISigner, Error> {
+        let client = HWIClient::get_client(device, false, chain)?;
+        Ok(HWISigner {
+            fingerprint: device.fingerprint,
+            client,
+        })
+    }
+}
+
+impl SignerCommon for HWISigner {
+    fn id(&self, _secp: &Secp256k1<All>) -> SignerId {
+        SignerId::Fingerprint(self.fingerprint)
+    }
+}
+
+impl TransactionSigner for HWISigner {
+    fn sign_transaction(
+        &self,
+        psbt: &mut PartiallySignedTransaction,
+        _sign_options: &bdk::SignOptions,
+        _secp: &Secp256k1<All>,
+    ) -> Result<(), SignerError> {
+        psbt.combine(
+            self.client
+                .sign_tx(psbt)
+                .map_err(|e| {
+                    SignerError::External(format!("While signing with hardware wallet: {}", e))
+                })?
+                .psbt,
+        )
+        .expect("Failed to combine HW signed psbt with passed PSBT");
+        Ok(())
+    }
+}
+
+// TODO: re-enable this once we have the `get_funded_wallet` test util
+// #[cfg(test)]
+// mod tests {
+//     #[test]
+//     fn test_hardware_signer() {
+//         use std::sync::Arc;
+//
+//         use bdk::tests::get_funded_wallet;
+//         use bdk::signer::SignerOrdering;
+//         use bdk::bitcoin::Network;
+//         use crate::HWISigner;
+//         use hwi::HWIClient;
+//
+//         let mut devices = HWIClient::enumerate().unwrap();
+//         if devices.is_empty() {
+//             panic!("No devices found!");
+//         }
+//         let device = devices.remove(0).unwrap();
+//         let client = HWIClient::get_client(&device, true, Network::Regtest.into()).unwrap();
+//         let descriptors = client.get_descriptors::<String>(None).unwrap();
+//         let custom_signer = HWISigner::from_device(&device, Network::Regtest.into()).unwrap();
+//
+//         let (mut wallet, _) = get_funded_wallet(&descriptors.internal[0]);
+//         wallet.add_signer(
+//             bdk::KeychainKind::External,
+//             SignerOrdering(200),
+//             Arc::new(custom_signer),
+//         );
+//
+//         let addr = wallet.get_address(bdk::wallet::AddressIndex::LastUnused);
+//         let mut builder = wallet.build_tx();
+//         builder.drain_to(addr.script_pubkey()).drain_wallet();
+//         let (mut psbt, _) = builder.finish().unwrap();
+//
+//         let finalized = wallet.sign(&mut psbt, Default::default()).unwrap();
+//         assert!(finalized);
+//     }
+// }

example-crates/example_bitcoind_rpc_polling/Cargo.toml

@@ -0,0 +1,12 @@
+[package]
+name = "example_bitcoind_rpc_polling"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+bdk_chain = { path = "../../crates/chain", features = ["serde"] }
+bdk_bitcoind_rpc = { path = "../../crates/bitcoind_rpc" }
+example_cli = { path = "../example_cli" }
+ctrlc = { version = "^2" }

example-crates/example_bitcoind_rpc_polling/README.md

@@ -0,0 +1,68 @@
+# Example RPC CLI
+
+### Simple Regtest Test
+
+1. Start local regtest bitcoind.
+   ```
+    mkdir -p /tmp/regtest/bitcoind
+    bitcoind -regtest -server -fallbackfee=0.0002 -rpcuser=<your-rpc-username> -rpcpassword=<your-rpc-password> -datadir=/tmp/regtest/bitcoind -daemon
+   ```
+2. Create a test bitcoind wallet and set bitcoind env.
+   ```
+   bitcoin-cli -datadir=/tmp/regtest/bitcoind -regtest -rpcuser=<your-rpc-username> -rpcpassword=<your-rpc-password> -named createwallet wallet_name="test"
+   export RPC_URL=127.0.0.1:18443
+   export RPC_USER=<your-rpc-username>
+   export RPC_PASS=<your-rpc-password>
+   ```
+3. Get test bitcoind wallet info.
+   ```
+   bitcoin-cli -rpcwallet="test" -rpcuser=<your-rpc-username> -rpcpassword=<your-rpc-password> -datadir=/tmp/regtest/bitcoind -regtest getwalletinfo
+   ```
+4. Get new test bitcoind wallet address.
+   ```
+   BITCOIND_ADDRESS=$(bitcoin-cli -rpcwallet="test" -datadir=/tmp/regtest/bitcoind -regtest -rpcuser=<your-rpc-username> -rpcpassword=<your-rpc-password> getnewaddress)
+   echo $BITCOIND_ADDRESS
+   ```
+5. Generate 101 blocks with reward to test bitcoind wallet address.
+   ```
+   bitcoin-cli -datadir=/tmp/regtest/bitcoind -regtest -rpcuser=<your-rpc-username> -rpcpassword=<your-rpc-password> generatetoaddress 101 $BITCOIND_ADDRESS
+   ```
+6. Verify test bitcoind wallet balance.
+   ```
+   bitcoin-cli -rpcwallet="test" -datadir=/tmp/regtest/bitcoind -regtest -rpcuser=<your-rpc-username> -rpcpassword=<your-rpc-password> getbalances
+   ```
+7. Set descriptor env and get address from RPC CLI wallet.
+   ```
+   export DESCRIPTOR="wpkh(tprv8ZgxMBicQKsPfK9BTf82oQkHhawtZv19CorqQKPFeaHDMA4dXYX6eWsJGNJ7VTQXWmoHdrfjCYuDijcRmNFwSKcVhswzqs4fugE8turndGc/1/*)"
+   cargo run -- --network regtest address next
+   ```
+8. Send 5 test bitcoin to RPC CLI wallet.
+   ```
+   bitcoin-cli -rpcwallet="test" -datadir=/tmp/regtest/bitcoind -regtest -rpcuser=<your-rpc-username> -rpcpassword=<your-rpc-password> sendtoaddress <address> 5
+   ```
+9. Sync blockchain with RPC CLI wallet.
+   ```
+   cargo run -- --network regtest sync
+   <CNTRL-C to stop syncing>
+   ```
+10. Get RPC CLI wallet unconfirmed balances.
+   ```
+   cargo run -- --network regtest balance
+   ```
+11. Generate 1 block with reward to test bitcoind wallet address.
+   ```
+   bitcoin-cli -datadir=/tmp/regtest/bitcoind -rpcuser=<your-rpc-username> -rpcpassword=<your-rpc-password> -regtest generatetoaddress 10 $BITCOIND_ADDRESS
+   ```
+12. Sync the blockchain with RPC CLI wallet.
+   ```
+   cargo run -- --network regtest sync
+   <CNTRL-C to stop syncing>
+   ```
+13. Get RPC CLI wallet confirmed balances.
+   ```
+   cargo run -- --network regtest balance
+   ```
+14. Get RPC CLI wallet transactions.
+   ```
+   cargo run -- --network regtest txout list
+   ```

example-crates/example_bitcoind_rpc_polling/src/main.rs

@@ -0,0 +1,390 @@
+use std::{
+    path::PathBuf,
+    sync::{
+        atomic::{AtomicBool, Ordering},
+        Arc, Mutex,
+    },
+    time::{Duration, Instant},
+};
+
+use bdk_bitcoind_rpc::{
+    bitcoincore_rpc::{Auth, Client, RpcApi},
+    Emitter,
+};
+use bdk_chain::{
+    bitcoin::{constants::genesis_block, Block, Transaction},
+    indexed_tx_graph, keychain,
+    local_chain::{self, LocalChain},
+    ConfirmationTimeHeightAnchor, IndexedTxGraph,
+};
+use example_cli::{
+    anyhow,
+    clap::{self, Args, Subcommand},
+    Keychain,
+};
+
+const DB_MAGIC: &[u8] = b"bdk_example_rpc";
+const DB_PATH: &str = ".bdk_example_rpc.db";
+
+/// The mpsc channel bound for emissions from [`Emitter`].
+const CHANNEL_BOUND: usize = 10;
+/// Delay for printing status to stdout.
+const STDOUT_PRINT_DELAY: Duration = Duration::from_secs(6);
+/// Delay between mempool emissions.
+const MEMPOOL_EMIT_DELAY: Duration = Duration::from_secs(30);
+/// Delay for committing to persistence.
+const DB_COMMIT_DELAY: Duration = Duration::from_secs(60);
+
+type ChangeSet = (
+    local_chain::ChangeSet,
+    indexed_tx_graph::ChangeSet<ConfirmationTimeHeightAnchor, keychain::ChangeSet<Keychain>>,
+);
+
+#[derive(Debug)]
+enum Emission {
+    Block(bdk_bitcoind_rpc::BlockEvent<Block>),
+    Mempool(Vec<(Transaction, u64)>),
+    Tip(u32),
+}
+
+#[derive(Args, Debug, Clone)]
+struct RpcArgs {
+    /// RPC URL
+    #[clap(env = "RPC_URL", long, default_value = "127.0.0.1:8332")]
+    url: String,
+    /// RPC auth cookie file
+    #[clap(env = "RPC_COOKIE", long)]
+    rpc_cookie: Option<PathBuf>,
+    /// RPC auth username
+    #[clap(env = "RPC_USER", long)]
+    rpc_user: Option<String>,
+    /// RPC auth password
+    #[clap(env = "RPC_PASS", long)]
+    rpc_password: Option<String>,
+    /// Starting block height to fallback to if no point of agreement if found
+    #[clap(env = "FALLBACK_HEIGHT", long, default_value = "0")]
+    fallback_height: u32,
+}
+
+impl From<RpcArgs> for Auth {
+    fn from(args: RpcArgs) -> Self {
+        match (args.rpc_cookie, args.rpc_user, args.rpc_password) {
+            (None, None, None) => Self::None,
+            (Some(path), _, _) => Self::CookieFile(path),
+            (_, Some(user), Some(pass)) => Self::UserPass(user, pass),
+            (_, Some(_), None) => panic!("rpc auth: missing rpc_pass"),
+            (_, None, Some(_)) => panic!("rpc auth: missing rpc_user"),
+        }
+    }
+}
+
+impl RpcArgs {
+    fn new_client(&self) -> anyhow::Result<Client> {
+        Ok(Client::new(
+            &self.url,
+            match (&self.rpc_cookie, &self.rpc_user, &self.rpc_password) {
+                (None, None, None) => Auth::None,
+                (Some(path), _, _) => Auth::CookieFile(path.clone()),
+                (_, Some(user), Some(pass)) => Auth::UserPass(user.clone(), pass.clone()),
+                (_, Some(_), None) => panic!("rpc auth: missing rpc_pass"),
+                (_, None, Some(_)) => panic!("rpc auth: missing rpc_user"),
+            },
+        )?)
+    }
+}
+
+#[derive(Subcommand, Debug, Clone)]
+enum RpcCommands {
+    /// Syncs local state with remote state via RPC (starting from last point of agreement) and
+    /// stores/indexes relevant transactions
+    Sync {
+        #[clap(flatten)]
+        rpc_args: RpcArgs,
+    },
+    /// Sync by having the emitter logic in a separate thread
+    Live {
+        #[clap(flatten)]
+        rpc_args: RpcArgs,
+    },
+}
+
+fn main() -> anyhow::Result<()> {
+    let start = Instant::now();
+    let example_cli::Init {
+        args,
+        keymap,
+        index,
+        db,
+        init_changeset,
+    } = example_cli::init::<RpcCommands, RpcArgs, ChangeSet>(DB_MAGIC, DB_PATH)?;
+    println!(
+        "[{:>10}s] loaded initial changeset from db",
+        start.elapsed().as_secs_f32()
+    );
+    let (init_chain_changeset, init_graph_changeset) = init_changeset;
+
+    let graph = Mutex::new({
+        let mut graph = IndexedTxGraph::new(index);
+        graph.apply_changeset(init_graph_changeset);
+        graph
+    });
+    println!(
+        "[{:>10}s] loaded indexed tx graph from changeset",
+        start.elapsed().as_secs_f32()
+    );
+
+    let chain = Mutex::new(if init_chain_changeset.is_empty() {
+        let genesis_hash = genesis_block(args.network).block_hash();
+        let (chain, chain_changeset) = LocalChain::from_genesis_hash(genesis_hash);
+        let mut db = db.lock().unwrap();
+        db.stage((chain_changeset, Default::default()));
+        db.commit()?;
+        chain
+    } else {
+        LocalChain::from_changeset(init_chain_changeset)?
+    });
+    println!(
+        "[{:>10}s] loaded local chain from changeset",
+        start.elapsed().as_secs_f32()
+    );
+
+    let rpc_cmd = match args.command {
+        example_cli::Commands::ChainSpecific(rpc_cmd) => rpc_cmd,
+        general_cmd => {
+            return example_cli::handle_commands(
+                &graph,
+                &db,
+                &chain,
+                &keymap,
+                args.network,
+                |rpc_args, tx| {
+                    let client = rpc_args.new_client()?;
+                    client.send_raw_transaction(tx)?;
+                    Ok(())
+                },
+                general_cmd,
+            );
+        }
+    };
+
+    match rpc_cmd {
+        RpcCommands::Sync { rpc_args } => {
+            let RpcArgs {
+                fallback_height, ..
+            } = rpc_args;
+
+            let chain_tip = chain.lock().unwrap().tip();
+            let rpc_client = rpc_args.new_client()?;
+            let mut emitter = Emitter::new(&rpc_client, chain_tip, fallback_height);
+
+            let mut last_db_commit = Instant::now();
+            let mut last_print = Instant::now();
+
+            while let Some(emission) = emitter.next_block()? {
+                let height = emission.block_height();
+
+                let mut chain = chain.lock().unwrap();
+                let mut graph = graph.lock().unwrap();
+                let mut db = db.lock().unwrap();
+
+                let chain_changeset = chain
+                    .apply_update(local_chain::Update {
+                        tip: emission.checkpoint,
+                        introduce_older_blocks: false,
+                    })
+                    .expect("must always apply as we receive blocks in order from emitter");
+                let graph_changeset = graph.apply_block_relevant(&emission.block, height);
+                db.stage((chain_changeset, graph_changeset));
+
+                // commit staged db changes in intervals
+                if last_db_commit.elapsed() >= DB_COMMIT_DELAY {
+                    last_db_commit = Instant::now();
+                    db.commit()?;
+                    println!(
+                        "[{:>10}s] committed to db (took {}s)",
+                        start.elapsed().as_secs_f32(),
+                        last_db_commit.elapsed().as_secs_f32()
+                    );
+                }
+
+                // print synced-to height and current balance in intervals
+                if last_print.elapsed() >= STDOUT_PRINT_DELAY {
+                    last_print = Instant::now();
+                    let synced_to = chain.tip();
+                    let balance = {
+                        graph.graph().balance(
+                            &*chain,
+                            synced_to.block_id(),
+                            graph.index.outpoints().iter().cloned(),
+                            |(k, _), _| k == &Keychain::Internal,
+                        )
+                    };
+                    println!(
+                        "[{:>10}s] synced to {} @ {} | total: {} sats",
+                        start.elapsed().as_secs_f32(),
+                        synced_to.hash(),
+                        synced_to.height(),
+                        balance.total()
+                    );
+                }
+            }
+
+            let mempool_txs = emitter.mempool()?;
+            let graph_changeset = graph.lock().unwrap().batch_insert_relevant_unconfirmed(
+                mempool_txs.iter().map(|(tx, time)| (tx, *time)),
+            );
+            {
+                let mut db = db.lock().unwrap();
+                db.stage((local_chain::ChangeSet::default(), graph_changeset));
+                db.commit()?; // commit one last time
+            }
+        }
+        RpcCommands::Live { rpc_args } => {
+            let RpcArgs {
+                fallback_height, ..
+            } = rpc_args;
+            let sigterm_flag = start_ctrlc_handler();
+
+            let last_cp = chain.lock().unwrap().tip();
+
+            println!(
+                "[{:>10}s] starting emitter thread...",
+                start.elapsed().as_secs_f32()
+            );
+            let (tx, rx) = std::sync::mpsc::sync_channel::<Emission>(CHANNEL_BOUND);
+            let emission_jh = std::thread::spawn(move || -> anyhow::Result<()> {
+                let rpc_client = rpc_args.new_client()?;
+                let mut emitter = Emitter::new(&rpc_client, last_cp, fallback_height);
+
+                let mut block_count = rpc_client.get_block_count()? as u32;
+                tx.send(Emission::Tip(block_count))?;
+
+                loop {
+                    match emitter.next_block()? {
+                        Some(block_emission) => {
+                            let height = block_emission.block_height();
+                            if sigterm_flag.load(Ordering::Acquire) {
+                                break;
+                            }
+                            if height > block_count {
+                                block_count = rpc_client.get_block_count()? as u32;
+                                tx.send(Emission::Tip(block_count))?;
+                            }
+                            tx.send(Emission::Block(block_emission))?;
+                        }
+                        None => {
+                            if await_flag(&sigterm_flag, MEMPOOL_EMIT_DELAY) {
+                                break;
+                            }
+                            println!("preparing mempool emission...");
+                            let now = Instant::now();
+                            tx.send(Emission::Mempool(emitter.mempool()?))?;
+                            println!("mempool emission prepared in {}s", now.elapsed().as_secs());
+                            continue;
+                        }
+                    };
+                }
+
+                println!("emitter thread shutting down...");
+                Ok(())
+            });
+
+            let mut tip_height = 0_u32;
+            let mut last_db_commit = Instant::now();
+            let mut last_print = Option::<Instant>::None;
+
+            for emission in rx {
+                let mut db = db.lock().unwrap();
+                let mut graph = graph.lock().unwrap();
+                let mut chain = chain.lock().unwrap();
+
+                let changeset = match emission {
+                    Emission::Block(block_emission) => {
+                        let height = block_emission.block_height();
+                        let chain_update = local_chain::Update {
+                            tip: block_emission.checkpoint,
+                            introduce_older_blocks: false,
+                        };
+                        let chain_changeset = chain
+                            .apply_update(chain_update)
+                            .expect("must always apply as we receive blocks in order from emitter");
+                        let graph_changeset =
+                            graph.apply_block_relevant(&block_emission.block, height);
+                        (chain_changeset, graph_changeset)
+                    }
+                    Emission::Mempool(mempool_txs) => {
+                        let graph_changeset = graph.batch_insert_relevant_unconfirmed(
+                            mempool_txs.iter().map(|(tx, time)| (tx, *time)),
+                        );
+                        (local_chain::ChangeSet::default(), graph_changeset)
+                    }
+                    Emission::Tip(h) => {
+                        tip_height = h;
+                        continue;
+                    }
+                };
+
+                db.stage(changeset);
+
+                if last_db_commit.elapsed() >= DB_COMMIT_DELAY {
+                    last_db_commit = Instant::now();
+                    db.commit()?;
+                    println!(
+                        "[{:>10}s] committed to db (took {}s)",
+                        start.elapsed().as_secs_f32(),
+                        last_db_commit.elapsed().as_secs_f32()
+                    );
+                }
+
+                if last_print.map_or(Duration::MAX, |i| i.elapsed()) >= STDOUT_PRINT_DELAY {
+                    last_print = Some(Instant::now());
+                    let synced_to = chain.tip();
+                    let balance = {
+                        graph.graph().balance(
+                            &*chain,
+                            synced_to.block_id(),
+                            graph.index.outpoints().iter().cloned(),
+                            |(k, _), _| k == &Keychain::Internal,
+                        )
+                    };
+                    println!(
+                        "[{:>10}s] synced to {} @ {} / {} | total: {} sats",
+                        start.elapsed().as_secs_f32(),
+                        synced_to.hash(),
+                        synced_to.height(),
+                        tip_height,
+                        balance.total()
+                    );
+                }
+            }
+
+            emission_jh.join().expect("must join emitter thread")?;
+        }
+    }
+
+    Ok(())
+}
+
+#[allow(dead_code)]
+fn start_ctrlc_handler() -> Arc<AtomicBool> {
+    let flag = Arc::new(AtomicBool::new(false));
+    let cloned_flag = flag.clone();
+
+    ctrlc::set_handler(move || cloned_flag.store(true, Ordering::Release));
+
+    flag
+}
+
+#[allow(dead_code)]
+fn await_flag(flag: &AtomicBool, duration: Duration) -> bool {
+    let start = Instant::now();
+    loop {
+        if flag.load(Ordering::Acquire) {
+            return true;
+        }
+        if start.elapsed() >= duration {
+            return false;
+        }
+        std::thread::sleep(Duration::from_secs(1));
+    }
+}

example-crates/example_cli/src/lib.rs

@@ -7,11 +7,12 @@ use std::{cmp::Reverse, collections::HashMap, path::PathBuf, sync::Mutex, time::
 
 use bdk_chain::{
     bitcoin::{
-        psbt::Prevouts, secp256k1::Secp256k1, util::sighash::SighashCache, Address, LockTime,
+        absolute, address, psbt::Prevouts, secp256k1::Secp256k1, sighash::SighashCache, Address,
         Network, Sequence, Transaction, TxIn, TxOut,
     },
-    indexed_tx_graph::{IndexedAdditions, IndexedTxGraph},
-    keychain::{DerivationAdditions, KeychainTxOutIndex},
+    indexed_tx_graph::{self, IndexedTxGraph},
+    keychain::{self, KeychainTxOutIndex},
+    local_chain,
     miniscript::{
         descriptor::{DescriptorSecretKey, KeyMap},
         Descriptor, DescriptorPublicKey,
@@ -24,13 +25,16 @@ pub use clap;
 use clap::{Parser, Subcommand};
 
 pub type KeychainTxGraph<A> = IndexedTxGraph<A, KeychainTxOutIndex<Keychain>>;
-pub type KeychainAdditions<A> = IndexedAdditions<A, DerivationAdditions<Keychain>>;
-pub type Database<'m, C> = Persist<Store<'m, C>, C>;
+pub type KeychainChangeSet<A> = (
+    local_chain::ChangeSet,
+    indexed_tx_graph::ChangeSet<A, keychain::ChangeSet<Keychain>>,
+);
+pub type Database<C> = Persist<Store<C>, C>;
 
 #[derive(Parser)]
 #[clap(author, version, about, long_about = None)]
 #[clap(propagate_version = true)]
-pub struct Args<S: clap::Subcommand> {
+pub struct Args<CS: clap::Subcommand, S: clap::Args> {
     #[clap(env = "DESCRIPTOR")]
     pub descriptor: String,
     #[clap(env = "CHANGE_DESCRIPTOR")]
@@ -46,14 +50,13 @@ pub struct Args<S: clap::Subcommand> {
     pub cp_limit: usize,
 
     #[clap(subcommand)]
-    pub command: Commands<S>,
+    pub command: Commands<CS, S>,
 }
 
-#[allow(clippy::almost_swapped)]
 #[derive(Subcommand, Debug, Clone)]
-pub enum Commands<S: clap::Subcommand> {
+pub enum Commands<CS: clap::Subcommand, S: clap::Args> {
     #[clap(flatten)]
-    ChainSpecific(S),
+    ChainSpecific(CS),
     /// Address generation and inspection.
     Address {
         #[clap(subcommand)]
@@ -69,10 +72,14 @@ pub enum Commands<S: clap::Subcommand> {
     },
     /// Send coins to an address.
     Send {
+        /// Amount to send in satoshis
         value: u64,
-        address: Address,
+        /// Destination address
+        address: Address<address::NetworkUnchecked>,
         #[clap(short, default_value = "bnb")]
         coin_select: CoinSelectionAlgo,
+        #[clap(flatten)]
+        chain_specific: S,
     },
 }
 
@@ -129,7 +136,6 @@ impl core::fmt::Display for CoinSelectionAlgo {
     }
 }
 
-#[allow(clippy::almost_swapped)]
 #[derive(Subcommand, Debug, Clone)]
 pub enum AddressCmd {
     /// Get the next unused address.
@@ -138,14 +144,17 @@ pub enum AddressCmd {
     New,
     /// List all addresses
     List {
+        /// List change addresses
         #[clap(long)]
         change: bool,
     },
+    /// Get last revealed address index for each keychain.
     Index,
 }
 
 #[derive(Subcommand, Debug, Clone)]
 pub enum TxOutCmd {
+    /// List transaction outputs.
     List {
         /// Return only spent outputs.
         #[clap(short, long)]
@@ -179,217 +188,12 @@ impl core::fmt::Display for Keychain {
     }
 }
 
-pub fn run_address_cmd<A, C>(
-    graph: &mut KeychainTxGraph<A>,
-    db: &Mutex<Database<C>>,
-    network: Network,
-    cmd: AddressCmd,
-) -> anyhow::Result<()>
-where
-    C: Default + Append + DeserializeOwned + Serialize + From<KeychainAdditions<A>>,
-{
-    let index = &mut graph.index;
-
-    match cmd {
-        AddressCmd::Next | AddressCmd::New => {
-            let spk_chooser = match cmd {
-                AddressCmd::Next => KeychainTxOutIndex::next_unused_spk,
-                AddressCmd::New => KeychainTxOutIndex::reveal_next_spk,
-                _ => unreachable!("only these two variants exist in match arm"),
-            };
-
-            let ((spk_i, spk), index_additions) = spk_chooser(index, &Keychain::External);
-            let db = &mut *db.lock().unwrap();
-            db.stage(C::from(KeychainAdditions::from(index_additions)));
-            db.commit()?;
-            let addr = Address::from_script(spk, network).context("failed to derive address")?;
-            println!("[address @ {}] {}", spk_i, addr);
-            Ok(())
-        }
-        AddressCmd::Index => {
-            for (keychain, derivation_index) in index.last_revealed_indices() {
-                println!("{:?}: {}", keychain, derivation_index);
-            }
-            Ok(())
-        }
-        AddressCmd::List { change } => {
-            let target_keychain = match change {
-                true => Keychain::Internal,
-                false => Keychain::External,
-            };
-            for (spk_i, spk) in index.revealed_spks_of_keychain(&target_keychain) {
-                let address = Address::from_script(spk, network)
-                    .expect("should always be able to derive address");
-                println!(
-                    "{:?} {} used:{}",
-                    spk_i,
-                    address,
-                    index.is_used(&(target_keychain, spk_i))
-                );
-            }
-            Ok(())
-        }
-    }
+pub struct CreateTxChange {
+    pub index_changeset: keychain::ChangeSet<Keychain>,
+    pub change_keychain: Keychain,
+    pub index: u32,
 }
 
-pub fn run_balance_cmd<A: Anchor, O: ChainOracle>(
-    graph: &KeychainTxGraph<A>,
-    chain: &O,
-) -> Result<(), O::Error> {
-    fn print_balances<'a>(title_str: &'a str, items: impl IntoIterator<Item = (&'a str, u64)>) {
-        println!("{}:", title_str);
-        for (name, amount) in items.into_iter() {
-            println!("    {:<10} {:>12} sats", name, amount)
-        }
-    }
-
-    let balance = graph.graph().try_balance(
-        chain,
-        chain.get_chain_tip()?.unwrap_or_default(),
-        graph.index.outpoints().iter().cloned(),
-        |(k, _), _| k == &Keychain::Internal,
-    )?;
-
-    let confirmed_total = balance.confirmed + balance.immature;
-    let unconfirmed_total = balance.untrusted_pending + balance.trusted_pending;
-
-    print_balances(
-        "confirmed",
-        [
-            ("total", confirmed_total),
-            ("spendable", balance.confirmed),
-            ("immature", balance.immature),
-        ],
-    );
-    print_balances(
-        "unconfirmed",
-        [
-            ("total", unconfirmed_total),
-            ("trusted", balance.trusted_pending),
-            ("untrusted", balance.untrusted_pending),
-        ],
-    );
-
-    Ok(())
-}
-
-pub fn run_txo_cmd<A: Anchor, O: ChainOracle>(
-    graph: &KeychainTxGraph<A>,
-    chain: &O,
-    network: Network,
-    cmd: TxOutCmd,
-) -> anyhow::Result<()>
-where
-    O::Error: std::error::Error + Send + Sync + 'static,
-{
-    let chain_tip = chain.get_chain_tip()?.unwrap_or_default();
-    let outpoints = graph.index.outpoints().iter().cloned();
-
-    match cmd {
-        TxOutCmd::List {
-            spent,
-            unspent,
-            confirmed,
-            unconfirmed,
-        } => {
-            let txouts = graph
-                .graph()
-                .try_filter_chain_txouts(chain, chain_tip, outpoints)
-                .filter(|r| match r {
-                    Ok((_, full_txo)) => match (spent, unspent) {
-                        (true, false) => full_txo.spent_by.is_some(),
-                        (false, true) => full_txo.spent_by.is_none(),
-                        _ => true,
-                    },
-                    // always keep errored items
-                    Err(_) => true,
-                })
-                .filter(|r| match r {
-                    Ok((_, full_txo)) => match (confirmed, unconfirmed) {
-                        (true, false) => full_txo.chain_position.is_confirmed(),
-                        (false, true) => !full_txo.chain_position.is_confirmed(),
-                        _ => true,
-                    },
-                    // always keep errored items
-                    Err(_) => true,
-                })
-                .collect::<Result<Vec<_>, _>>()?;
-
-            for (spk_i, full_txo) in txouts {
-                let addr = Address::from_script(&full_txo.txout.script_pubkey, network)?;
-                println!(
-                    "{:?} {} {} {} spent:{:?}",
-                    spk_i, full_txo.txout.value, full_txo.outpoint, addr, full_txo.spent_by
-                )
-            }
-            Ok(())
-        }
-    }
-}
-
-#[allow(clippy::too_many_arguments)]
-pub fn run_send_cmd<A: Anchor, O: ChainOracle, C>(
-    graph: &Mutex<KeychainTxGraph<A>>,
-    db: &Mutex<Database<'_, C>>,
-    chain: &O,
-    keymap: &HashMap<DescriptorPublicKey, DescriptorSecretKey>,
-    cs_algorithm: CoinSelectionAlgo,
-    address: Address,
-    value: u64,
-    broadcast: impl FnOnce(&Transaction) -> anyhow::Result<()>,
-) -> anyhow::Result<()>
-where
-    O::Error: std::error::Error + Send + Sync + 'static,
-    C: Default + Append + DeserializeOwned + Serialize + From<KeychainAdditions<A>>,
-{
-    let (transaction, change_index) = {
-        let graph = &mut *graph.lock().unwrap();
-        // take mutable ref to construct tx -- it is only open for a short time while building it.
-        let (tx, change_info) = create_tx(graph, chain, keymap, cs_algorithm, address, value)?;
-
-        if let Some((index_additions, (change_keychain, index))) = change_info {
-            // We must first persist to disk the fact that we've got a new address from the
-            // change keychain so future scans will find the tx we're about to broadcast.
-            // If we're unable to persist this, then we don't want to broadcast.
-            {
-                let db = &mut *db.lock().unwrap();
-                db.stage(C::from(KeychainAdditions::from(index_additions)));
-                db.commit()?;
-            }
-
-            // We don't want other callers/threads to use this address while we're using it
-            // but we also don't want to scan the tx we just created because it's not
-            // technically in the blockchain yet.
-            graph.index.mark_used(&change_keychain, index);
-            (tx, Some((change_keychain, index)))
-        } else {
-            (tx, None)
-        }
-    };
-
-    match (broadcast)(&transaction) {
-        Ok(_) => {
-            println!("Broadcasted Tx : {}", transaction.txid());
-
-            let keychain_additions = graph.lock().unwrap().insert_tx(&transaction, None, None);
-
-            // We know the tx is at least unconfirmed now. Note if persisting here fails,
-            // it's not a big deal since we can always find it again form
-            // blockchain.
-            db.lock().unwrap().stage(C::from(keychain_additions));
-            Ok(())
-        }
-        Err(e) => {
-            if let Some((keychain, index)) = change_index {
-                // We failed to broadcast, so allow our change address to be used in the future
-                graph.lock().unwrap().index.unmark_used(&keychain, index);
-            }
-            Err(e)
-        }
-    }
-}
-
-#[allow(clippy::type_complexity)]
 pub fn create_tx<A: Anchor, O: ChainOracle>(
     graph: &mut KeychainTxGraph<A>,
     chain: &O,
@@ -397,14 +201,11 @@ pub fn create_tx<A: Anchor, O: ChainOracle>(
     cs_algorithm: CoinSelectionAlgo,
     address: Address,
     value: u64,
-) -> anyhow::Result<(
-    Transaction,
-    Option<(DerivationAdditions<Keychain>, (Keychain, u32))>,
-)>
+) -> anyhow::Result<(Transaction, Option<CreateTxChange>)>
 where
     O::Error: std::error::Error + Send + Sync + 'static,
 {
-    let mut additions = DerivationAdditions::default();
+    let mut changeset = keychain::ChangeSet::default();
 
     let assets = bdk_tmp_plan::Assets {
         keys: keymap.iter().map(|(pk, _)| pk.clone()).collect(),
@@ -452,12 +253,12 @@ where
         Keychain::External
     };
 
-    let ((change_index, change_script), change_additions) =
+    let ((change_index, change_script), change_changeset) =
         graph.index.next_unused_spk(&internal_keychain);
-    additions.append(change_additions);
+    changeset.append(change_changeset);
 
     // Clone to drop the immutable reference.
-    let change_script = change_script.clone();
+    let change_script = change_script.into();
 
     let change_plan = bdk_tmp_plan::plan_satisfaction(
         &graph
@@ -465,7 +266,8 @@ where
             .keychains()
             .get(&internal_keychain)
             .expect("must exist")
-            .at_derivation_index(change_index),
+            .at_derivation_index(change_index)
+            .expect("change_index can't be hardened"),
         &assets,
     )
     .expect("failed to obtain change plan");
@@ -518,11 +320,8 @@ where
         version: 0x02,
         // because the temporary planning module does not support timelocks, we can use the chain
         // tip as the `lock_time` for anti-fee-sniping purposes
-        lock_time: chain
-            .get_chain_tip()?
-            .and_then(|block_id| LockTime::from_height(block_id.height).ok())
-            .unwrap_or(LockTime::ZERO)
-            .into(),
+        lock_time: absolute::LockTime::from_height(chain.get_chain_tip()?.height)
+            .expect("invalid height"),
         input: selected_txos
             .iter()
             .map(|(_, utxo)| TxIn {
@@ -594,7 +393,11 @@ where
     }
 
     let change_info = if selection_meta.drain_value.is_some() {
-        Some((additions, (internal_keychain, change_index)))
+        Some(CreateTxChange {
+            index_changeset: changeset,
+            change_keychain: internal_keychain,
+            index: change_index,
+        })
     } else {
         None
     };
@@ -602,104 +405,284 @@ where
     Ok((transaction, change_info))
 }
 
-#[allow(clippy::type_complexity)]
+// Alias the elements of `Result` of `planned_utxos`
+pub type PlannedUtxo<K, A> = (bdk_tmp_plan::Plan<K>, FullTxOut<A>);
+
 pub fn planned_utxos<A: Anchor, O: ChainOracle, K: Clone + bdk_tmp_plan::CanDerive>(
     graph: &KeychainTxGraph<A>,
     chain: &O,
     assets: &bdk_tmp_plan::Assets<K>,
-) -> Result<Vec<(bdk_tmp_plan::Plan<K>, FullTxOut<A>)>, O::Error> {
-    let chain_tip = chain.get_chain_tip()?.unwrap_or_default();
+) -> Result<Vec<PlannedUtxo<K, A>>, O::Error> {
+    let chain_tip = chain.get_chain_tip()?;
     let outpoints = graph.index.outpoints().iter().cloned();
     graph
         .graph()
         .try_filter_chain_unspents(chain, chain_tip, outpoints)
-        .filter_map(
-            #[allow(clippy::type_complexity)]
-            |r| -> Option<Result<(bdk_tmp_plan::Plan<K>, FullTxOut<A>), _>> {
-                let (k, i, full_txo) = match r {
-                    Err(err) => return Some(Err(err)),
-                    Ok(((k, i), full_txo)) => (k, i, full_txo),
-                };
-                let desc = graph
-                    .index
-                    .keychains()
-                    .get(&k)
-                    .expect("keychain must exist")
-                    .at_derivation_index(i);
-                let plan = bdk_tmp_plan::plan_satisfaction(&desc, assets)?;
-                Some(Ok((plan, full_txo)))
-            },
-        )
+        .filter_map(|r| -> Option<Result<PlannedUtxo<K, A>, _>> {
+            let (k, i, full_txo) = match r {
+                Err(err) => return Some(Err(err)),
+                Ok(((k, i), full_txo)) => (k, i, full_txo),
+            };
+            let desc = graph
+                .index
+                .keychains()
+                .get(&k)
+                .expect("keychain must exist")
+                .at_derivation_index(i)
+                .expect("i can't be hardened");
+            let plan = bdk_tmp_plan::plan_satisfaction(&desc, assets)?;
+            Some(Ok((plan, full_txo)))
+        })
         .collect()
 }
 
-pub fn handle_commands<S: clap::Subcommand, A: Anchor, O: ChainOracle, C>(
+pub fn handle_commands<CS: clap::Subcommand, S: clap::Args, A: Anchor, O: ChainOracle, C>(
     graph: &Mutex<KeychainTxGraph<A>>,
     db: &Mutex<Database<C>>,
     chain: &Mutex<O>,
     keymap: &HashMap<DescriptorPublicKey, DescriptorSecretKey>,
     network: Network,
-    broadcast: impl FnOnce(&Transaction) -> anyhow::Result<()>,
-    cmd: Commands<S>,
+    broadcast: impl FnOnce(S, &Transaction) -> anyhow::Result<()>,
+    cmd: Commands<CS, S>,
 ) -> anyhow::Result<()>
 where
     O::Error: std::error::Error + Send + Sync + 'static,
-    C: Default + Append + DeserializeOwned + Serialize + From<KeychainAdditions<A>>,
+    C: Default + Append + DeserializeOwned + Serialize + From<KeychainChangeSet<A>>,
 {
     match cmd {
         Commands::ChainSpecific(_) => unreachable!("example code should handle this!"),
         Commands::Address { addr_cmd } => {
             let graph = &mut *graph.lock().unwrap();
-            run_address_cmd(graph, db, network, addr_cmd)
+            let index = &mut graph.index;
+
+            match addr_cmd {
+                AddressCmd::Next | AddressCmd::New => {
+                    let spk_chooser = match addr_cmd {
+                        AddressCmd::Next => KeychainTxOutIndex::next_unused_spk,
+                        AddressCmd::New => KeychainTxOutIndex::reveal_next_spk,
+                        _ => unreachable!("only these two variants exist in match arm"),
+                    };
+
+                    let ((spk_i, spk), index_changeset) = spk_chooser(index, &Keychain::External);
+                    let db = &mut *db.lock().unwrap();
+                    db.stage_and_commit(C::from((
+                        local_chain::ChangeSet::default(),
+                        indexed_tx_graph::ChangeSet::from(index_changeset),
+                    )))?;
+                    let addr =
+                        Address::from_script(spk, network).context("failed to derive address")?;
+                    println!("[address @ {}] {}", spk_i, addr);
+                    Ok(())
+                }
+                AddressCmd::Index => {
+                    for (keychain, derivation_index) in index.last_revealed_indices() {
+                        println!("{:?}: {}", keychain, derivation_index);
+                    }
+                    Ok(())
+                }
+                AddressCmd::List { change } => {
+                    let target_keychain = match change {
+                        true => Keychain::Internal,
+                        false => Keychain::External,
+                    };
+                    for (spk_i, spk) in index.revealed_keychain_spks(&target_keychain) {
+                        let address = Address::from_script(spk, network)
+                            .expect("should always be able to derive address");
+                        println!(
+                            "{:?} {} used:{}",
+                            spk_i,
+                            address,
+                            index.is_used(target_keychain, spk_i)
+                        );
+                    }
+                    Ok(())
+                }
+            }
         }
         Commands::Balance => {
             let graph = &*graph.lock().unwrap();
             let chain = &*chain.lock().unwrap();
-            run_balance_cmd(graph, chain).map_err(anyhow::Error::from)
+            fn print_balances<'a>(
+                title_str: &'a str,
+                items: impl IntoIterator<Item = (&'a str, u64)>,
+            ) {
+                println!("{}:", title_str);
+                for (name, amount) in items.into_iter() {
+                    println!("    {:<10} {:>12} sats", name, amount)
+                }
+            }
+
+            let balance = graph.graph().try_balance(
+                chain,
+                chain.get_chain_tip()?,
+                graph.index.outpoints().iter().cloned(),
+                |(k, _), _| k == &Keychain::Internal,
+            )?;
+
+            let confirmed_total = balance.confirmed + balance.immature;
+            let unconfirmed_total = balance.untrusted_pending + balance.trusted_pending;
+
+            print_balances(
+                "confirmed",
+                [
+                    ("total", confirmed_total),
+                    ("spendable", balance.confirmed),
+                    ("immature", balance.immature),
+                ],
+            );
+            print_balances(
+                "unconfirmed",
+                [
+                    ("total", unconfirmed_total),
+                    ("trusted", balance.trusted_pending),
+                    ("untrusted", balance.untrusted_pending),
+                ],
+            );
+
+            Ok(())
         }
         Commands::TxOut { txout_cmd } => {
             let graph = &*graph.lock().unwrap();
             let chain = &*chain.lock().unwrap();
-            run_txo_cmd(graph, chain, network, txout_cmd)
+            let chain_tip = chain.get_chain_tip()?;
+            let outpoints = graph.index.outpoints().iter().cloned();
+
+            match txout_cmd {
+                TxOutCmd::List {
+                    spent,
+                    unspent,
+                    confirmed,
+                    unconfirmed,
+                } => {
+                    let txouts = graph
+                        .graph()
+                        .try_filter_chain_txouts(chain, chain_tip, outpoints)
+                        .filter(|r| match r {
+                            Ok((_, full_txo)) => match (spent, unspent) {
+                                (true, false) => full_txo.spent_by.is_some(),
+                                (false, true) => full_txo.spent_by.is_none(),
+                                _ => true,
+                            },
+                            // always keep errored items
+                            Err(_) => true,
+                        })
+                        .filter(|r| match r {
+                            Ok((_, full_txo)) => match (confirmed, unconfirmed) {
+                                (true, false) => full_txo.chain_position.is_confirmed(),
+                                (false, true) => !full_txo.chain_position.is_confirmed(),
+                                _ => true,
+                            },
+                            // always keep errored items
+                            Err(_) => true,
+                        })
+                        .collect::<Result<Vec<_>, _>>()?;
+
+                    for (spk_i, full_txo) in txouts {
+                        let addr = Address::from_script(&full_txo.txout.script_pubkey, network)?;
+                        println!(
+                            "{:?} {} {} {} spent:{:?}",
+                            spk_i, full_txo.txout.value, full_txo.outpoint, addr, full_txo.spent_by
+                        )
+                    }
+                    Ok(())
+                }
+            }
         }
         Commands::Send {
             value,
             address,
             coin_select,
+            chain_specific,
         } => {
             let chain = &*chain.lock().unwrap();
-            run_send_cmd(
-                graph,
-                db,
-                chain,
-                keymap,
-                coin_select,
-                address,
-                value,
-                broadcast,
-            )
+            let address = address.require_network(network)?;
+            let (transaction, change_index) = {
+                let graph = &mut *graph.lock().unwrap();
+                // take mutable ref to construct tx -- it is only open for a short time while building it.
+                let (tx, change_info) =
+                    create_tx(graph, chain, keymap, coin_select, address, value)?;
+
+                if let Some(CreateTxChange {
+                    index_changeset,
+                    change_keychain,
+                    index,
+                }) = change_info
+                {
+                    // We must first persist to disk the fact that we've got a new address from the
+                    // change keychain so future scans will find the tx we're about to broadcast.
+                    // If we're unable to persist this, then we don't want to broadcast.
+                    {
+                        let db = &mut *db.lock().unwrap();
+                        db.stage_and_commit(C::from((
+                            local_chain::ChangeSet::default(),
+                            indexed_tx_graph::ChangeSet::from(index_changeset),
+                        )))?;
+                    }
+
+                    // We don't want other callers/threads to use this address while we're using it
+                    // but we also don't want to scan the tx we just created because it's not
+                    // technically in the blockchain yet.
+                    graph.index.mark_used(change_keychain, index);
+                    (tx, Some((change_keychain, index)))
+                } else {
+                    (tx, None)
+                }
+            };
+
+            match (broadcast)(chain_specific, &transaction) {
+                Ok(_) => {
+                    println!("Broadcasted Tx : {}", transaction.txid());
+
+                    let keychain_changeset = graph.lock().unwrap().insert_tx(transaction);
+
+                    // We know the tx is at least unconfirmed now. Note if persisting here fails,
+                    // it's not a big deal since we can always find it again form
+                    // blockchain.
+                    db.lock().unwrap().stage_and_commit(C::from((
+                        local_chain::ChangeSet::default(),
+                        keychain_changeset,
+                    )))?;
+                    Ok(())
+                }
+                Err(e) => {
+                    if let Some((keychain, index)) = change_index {
+                        // We failed to broadcast, so allow our change address to be used in the future
+                        graph.lock().unwrap().index.unmark_used(keychain, index);
+                    }
+                    Err(e)
+                }
+            }
         }
     }
 }
 
-#[allow(clippy::type_complexity)]
-pub fn init<'m, S: clap::Subcommand, C>(
-    db_magic: &'m [u8],
+/// The initial state returned by [`init`].
+pub struct Init<CS: clap::Subcommand, S: clap::Args, C> {
+    /// Arguments parsed by the cli.
+    pub args: Args<CS, S>,
+    /// Descriptor keymap.
+    pub keymap: KeyMap,
+    /// Keychain-txout index.
+    pub index: KeychainTxOutIndex<Keychain>,
+    /// Persistence backend.
+    pub db: Mutex<Database<C>>,
+    /// Initial changeset.
+    pub init_changeset: C,
+}
+
+/// Parses command line arguments and initializes all components, creating
+/// a file store with the given parameters, or loading one if it exists.
+pub fn init<CS: clap::Subcommand, S: clap::Args, C>(
+    db_magic: &[u8],
     db_default_path: &str,
-) -> anyhow::Result<(
-    Args<S>,
-    KeyMap,
-    KeychainTxOutIndex<Keychain>,
-    Mutex<Database<'m, C>>,
-    C,
-)>
+) -> anyhow::Result<Init<CS, S, C>>
 where
     C: Default + Append + Serialize + DeserializeOwned,
 {
     if std::env::var("BDK_DB_PATH").is_err() {
         std::env::set_var("BDK_DB_PATH", db_default_path);
     }
-    let args = Args::<S>::parse();
+    let args = Args::<CS, S>::parse();
     let secp = Secp256k1::default();
 
     let mut index = KeychainTxOutIndex::<Keychain>::default();
@@ -718,19 +701,19 @@ where
         index.add_keychain(Keychain::Internal, internal_descriptor);
     }
 
-    let mut db_backend = match Store::<'m, C>::new_from_path(db_magic, &args.db_path) {
+    let mut db_backend = match Store::<C>::open_or_create_new(db_magic, &args.db_path) {
         Ok(db_backend) => db_backend,
         // we cannot return `err` directly as it has lifetime `'m`
         Err(err) => return Err(anyhow::anyhow!("failed to init db backend: {:?}", err)),
     };
 
-    let init_changeset = db_backend.load_from_persistence()?;
+    let init_changeset = db_backend.load_from_persistence()?.unwrap_or_default();
 
-    Ok((
+    Ok(Init {
         args,
         keymap,
         index,
-        Mutex::new(Database::new(db_backend)),
+        db: Mutex::new(Database::new(db_backend)),
         init_changeset,
-    ))
+    })
 }

example-crates/example_electrum/src/main.rs

@@ -5,14 +5,14 @@ use std::{
 };
 
 use bdk_chain::{
-    bitcoin::{Address, BlockHash, Network, OutPoint, Txid},
-    indexed_tx_graph::{IndexedAdditions, IndexedTxGraph},
-    keychain::LocalChangeSet,
-    local_chain::LocalChain,
+    bitcoin::{constants::genesis_block, Address, Network, OutPoint, Txid},
+    indexed_tx_graph::{self, IndexedTxGraph},
+    keychain,
+    local_chain::{self, LocalChain},
     Append, ConfirmationHeightAnchor,
 };
 use bdk_electrum::{
-    electrum_client::{self, ElectrumApi},
+    electrum_client::{self, Client, ElectrumApi},
     ElectrumExt, ElectrumUpdate,
 };
 use example_cli::{
@@ -22,8 +22,7 @@ use example_cli::{
 };
 
 const DB_MAGIC: &[u8] = b"bdk_example_electrum";
-const DB_PATH: &str = ".bdk_electrum_example.db";
-const ASSUME_FINAL_DEPTH: usize = 10;
+const DB_PATH: &str = ".bdk_example_electrum.db";
 
 #[derive(Subcommand, Debug, Clone)]
 enum ElectrumCommands {
@@ -34,6 +33,8 @@ enum ElectrumCommands {
         stop_gap: usize,
         #[clap(flatten)]
         scan_options: ScanOptions,
+        #[clap(flatten)]
+        electrum_args: ElectrumArgs,
     },
     /// Scans particular addresses using the electrum API.
     Sync {
@@ -51,9 +52,44 @@ enum ElectrumCommands {
         unconfirmed: bool,
         #[clap(flatten)]
         scan_options: ScanOptions,
+        #[clap(flatten)]
+        electrum_args: ElectrumArgs,
     },
 }
 
+impl ElectrumCommands {
+    fn electrum_args(&self) -> ElectrumArgs {
+        match self {
+            ElectrumCommands::Scan { electrum_args, .. } => electrum_args.clone(),
+            ElectrumCommands::Sync { electrum_args, .. } => electrum_args.clone(),
+        }
+    }
+}
+
+#[derive(clap::Args, Debug, Clone)]
+pub struct ElectrumArgs {
+    /// The electrum url to use to connect to. If not provided it will use a default electrum server
+    /// for your chosen network.
+    electrum_url: Option<String>,
+}
+
+impl ElectrumArgs {
+    pub fn client(&self, network: Network) -> anyhow::Result<Client> {
+        let electrum_url = self.electrum_url.as_deref().unwrap_or(match network {
+            Network::Bitcoin => "ssl://electrum.blockstream.info:50002",
+            Network::Testnet => "ssl://electrum.blockstream.info:60002",
+            Network::Regtest => "tcp://localhost:60401",
+            Network::Signet => "tcp://signet-electrumx.wakiyamap.dev:50001",
+            _ => panic!("Unknown network"),
+        });
+        let config = electrum_client::Config::builder()
+            .validate_domain(matches!(network, Network::Bitcoin))
+            .build();
+
+        Ok(electrum_client::Client::from_config(electrum_url, config)?)
+    }
+}
+
 #[derive(Parser, Debug, Clone, PartialEq)]
 pub struct ScanOptions {
     /// Set batch size for each script_history call to electrum client.
@@ -61,71 +97,69 @@ pub struct ScanOptions {
     pub batch_size: usize,
 }
 
-type ChangeSet = LocalChangeSet<Keychain, ConfirmationHeightAnchor>;
+type ChangeSet = (
+    local_chain::ChangeSet,
+    indexed_tx_graph::ChangeSet<ConfirmationHeightAnchor, keychain::ChangeSet<Keychain>>,
+);
 
 fn main() -> anyhow::Result<()> {
-    let (args, keymap, index, db, init_changeset) =
-        example_cli::init::<ElectrumCommands, ChangeSet>(DB_MAGIC, DB_PATH)?;
+    let example_cli::Init {
+        args,
+        keymap,
+        index,
+        db,
+        init_changeset,
+    } = example_cli::init::<ElectrumCommands, ElectrumArgs, ChangeSet>(DB_MAGIC, DB_PATH)?;
+
+    let (disk_local_chain, disk_tx_graph) = init_changeset;
 
     let graph = Mutex::new({
         let mut graph = IndexedTxGraph::new(index);
-        graph.apply_additions(init_changeset.indexed_additions);
+        graph.apply_changeset(disk_tx_graph);
         graph
     });
 
     let chain = Mutex::new({
-        let mut chain = LocalChain::default();
-        chain.apply_changeset(init_changeset.chain_changeset);
+        let genesis_hash = genesis_block(args.network).block_hash();
+        let (mut chain, _) = LocalChain::from_genesis_hash(genesis_hash);
+        chain.apply_changeset(&disk_local_chain)?;
         chain
     });
 
-    let electrum_url = match args.network {
-        Network::Bitcoin => "ssl://electrum.blockstream.info:50002",
-        Network::Testnet => "ssl://electrum.blockstream.info:60002",
-        Network::Regtest => "tcp://localhost:60401",
-        Network::Signet => "tcp://signet-electrumx.wakiyamap.dev:50001",
-    };
-    let config = electrum_client::Config::builder()
-        .validate_domain(matches!(args.network, Network::Bitcoin))
-        .build();
-
-    let client = electrum_client::Client::from_config(electrum_url, config)?;
-
     let electrum_cmd = match &args.command {
         example_cli::Commands::ChainSpecific(electrum_cmd) => electrum_cmd,
         general_cmd => {
-            let res = example_cli::handle_commands(
+            return example_cli::handle_commands(
                 &graph,
                 &db,
                 &chain,
                 &keymap,
                 args.network,
-                |tx| {
-                    client
-                        .transaction_broadcast(tx)
-                        .map(|_| ())
-                        .map_err(anyhow::Error::from)
+                |electrum_args, tx| {
+                    let client = electrum_args.client(args.network)?;
+                    client.transaction_broadcast(tx)?;
+                    Ok(())
                 },
                 general_cmd.clone(),
             );
-
-            db.lock().unwrap().commit()?;
-            return res;
         }
     };
 
+    let client = electrum_cmd.electrum_args().client(args.network)?;
+
     let response = match electrum_cmd.clone() {
         ElectrumCommands::Scan {
             stop_gap,
             scan_options,
+            ..
         } => {
-            let (keychain_spks, local_chain) = {
+            let (keychain_spks, tip) = {
                 let graph = &*graph.lock().unwrap();
                 let chain = &*chain.lock().unwrap();
 
                 let keychain_spks = graph
                     .index
-                    .spks_of_all_keychains()
+                    .all_unbounded_spk_iters()
                     .into_iter()
                     .map(|(keychain, iter)| {
                         let mut first = true;
@@ -142,26 +176,12 @@ fn main() -> anyhow::Result<()> {
                     })
                     .collect::<BTreeMap<_, _>>();
 
-                let c = chain
-                    .blocks()
-                    .iter()
-                    .rev()
-                    .take(ASSUME_FINAL_DEPTH)
-                    .map(|(k, v)| (*k, *v))
-                    .collect::<BTreeMap<u32, BlockHash>>();
-
-                (keychain_spks, c)
+                let tip = chain.tip();
+                (keychain_spks, tip)
             };
 
             client
-                .scan(
-                    &local_chain,
-                    keychain_spks,
-                    core::iter::empty(),
-                    core::iter::empty(),
-                    stop_gap,
-                    scan_options.batch_size,
-                )
+                .full_scan(tip, keychain_spks, stop_gap, scan_options.batch_size)
                 .context("scanning the blockchain")?
         }
         ElectrumCommands::Sync {
@@ -170,11 +190,12 @@ fn main() -> anyhow::Result<()> {
             mut utxos,
             mut unconfirmed,
             scan_options,
+            ..
         } => {
             // Get a short lock on the tracker to get the spks we're interested in
             let graph = graph.lock().unwrap();
             let chain = chain.lock().unwrap();
-            let chain_tip = chain.tip().unwrap_or_default();
+            let chain_tip = chain.tip().block_id();
 
             if !(all_spks || unused_spks || utxos || unconfirmed) {
                 unused_spks = true;
@@ -184,34 +205,33 @@ fn main() -> anyhow::Result<()> {
                 unused_spks = false;
             }
 
-            let mut spks: Box<dyn Iterator<Item = bdk_chain::bitcoin::Script>> =
+            let mut spks: Box<dyn Iterator<Item = bdk_chain::bitcoin::ScriptBuf>> =
                 Box::new(core::iter::empty());
             if all_spks {
                 let all_spks = graph
                     .index
-                    .all_spks()
-                    .iter()
-                    .map(|(k, v)| (*k, v.clone()))
+                    .revealed_spks()
+                    .map(|(k, i, spk)| (k, i, spk.to_owned()))
                     .collect::<Vec<_>>();
-                spks = Box::new(spks.chain(all_spks.into_iter().map(|(index, script)| {
-                    eprintln!("scanning {:?}", index);
-                    script
+                spks = Box::new(spks.chain(all_spks.into_iter().map(|(k, i, spk)| {
+                    eprintln!("scanning {}:{}", k, i);
+                    spk
                 })));
             }
             if unused_spks {
                 let unused_spks = graph
                     .index
-                    .unused_spks(..)
-                    .map(|(k, v)| (*k, v.clone()))
+                    .unused_spks()
+                    .map(|(k, i, spk)| (k, i, spk.to_owned()))
                     .collect::<Vec<_>>();
-                spks = Box::new(spks.chain(unused_spks.into_iter().map(|(index, script)| {
+                spks = Box::new(spks.chain(unused_spks.into_iter().map(|(k, i, spk)| {
                     eprintln!(
-                        "Checking if address {} {:?} has been used",
-                        Address::from_script(&script, args.network).unwrap(),
-                        index
+                        "Checking if address {} {}:{} has been used",
+                        Address::from_script(&spk, args.network).unwrap(),
+                        k,
+                        i,
                     );
-
-                    script
+                    spk
                 })));
             }
 
@@ -245,8 +265,8 @@ fn main() -> anyhow::Result<()> {
                 let unconfirmed_txids = graph
                     .graph()
                     .list_chain_txs(&*chain, chain_tip)
-                    .filter(|canonical_tx| !canonical_tx.observed_as.is_confirmed())
-                    .map(|canonical_tx| canonical_tx.node.txid)
+                    .filter(|canonical_tx| !canonical_tx.chain_position.is_confirmed())
+                    .map(|canonical_tx| canonical_tx.tx_node.txid)
                     .collect::<Vec<Txid>>();
 
                 txids = Box::new(unconfirmed_txids.into_iter().inspect(|txid| {
@@ -254,31 +274,29 @@ fn main() -> anyhow::Result<()> {
                 }));
             }
 
-            let c = chain
-                .blocks()
-                .iter()
-                .rev()
-                .take(ASSUME_FINAL_DEPTH)
-                .map(|(k, v)| (*k, *v))
-                .collect::<BTreeMap<u32, BlockHash>>();
+            let tip = chain.tip();
 
             // drop lock on graph and chain
             drop((graph, chain));
 
-            let update = client
-                .scan_without_keychain(&c, spks, txids, outpoints, scan_options.batch_size)
+            let electrum_update = client
+                .sync(tip, spks, txids, outpoints, scan_options.batch_size)
                 .context("scanning the blockchain")?;
-            ElectrumUpdate {
-                graph_update: update.graph_update,
-                chain_update: update.chain_update,
-                keychain_update: BTreeMap::new(),
-            }
+            (electrum_update, BTreeMap::new())
         }
     };
 
+    let (
+        ElectrumUpdate {
+            chain_update,
+            relevant_txids,
+        },
+        keychain_update,
+    ) = response;
+
     let missing_txids = {
         let graph = &*graph.lock().unwrap();
-        response.missing_full_txs(graph.graph())
+        relevant_txids.missing_full_txs(graph.graph())
     };
 
     let now = std::time::UNIX_EPOCH
@@ -286,29 +304,27 @@ fn main() -> anyhow::Result<()> {
         .expect("must get time")
         .as_secs();
 
-    let final_update = response.finalize(&client, Some(now), missing_txids)?;
+    let graph_update = relevant_txids.into_tx_graph(&client, Some(now), missing_txids)?;
 
     let db_changeset = {
         let mut chain = chain.lock().unwrap();
         let mut graph = graph.lock().unwrap();
 
-        let chain_changeset = chain.apply_update(final_update.chain)?;
+        let chain = chain.apply_update(chain_update)?;
 
-        let indexed_additions = {
-            let mut additions = IndexedAdditions::<ConfirmationHeightAnchor, _>::default();
-            let (_, index_additions) = graph.index.reveal_to_target_multi(&final_update.keychain);
-            additions.append(IndexedAdditions {
-                index_additions,
+        let indexed_tx_graph = {
+            let mut changeset =
+                indexed_tx_graph::ChangeSet::<ConfirmationHeightAnchor, _>::default();
+            let (_, indexer) = graph.index.reveal_to_target_multi(&keychain_update);
+            changeset.append(indexed_tx_graph::ChangeSet {
+                indexer,
                 ..Default::default()
             });
-            additions.append(graph.apply_update(final_update.graph));
-            additions
+            changeset.append(graph.apply_update(graph_update));
+            changeset
         };
 
-        ChangeSet {
-            indexed_additions,
-            chain_changeset,
-        }
+        (chain, indexed_tx_graph)
     };
 
     let mut db = db.lock().unwrap();

example-crates/example_esplora/Cargo.toml

@@ -0,0 +1,12 @@
+[package]
+name = "example_esplora"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+bdk_chain = { path = "../../crates/chain", features = ["serde"] }
+bdk_esplora = { path = "../../crates/esplora", features = ["blocking"] }
+example_cli = { path = "../example_cli" }
+

example-crates/example_esplora/src/main.rs

@@ -0,0 +1,350 @@
+use std::{
+    collections::{BTreeMap, BTreeSet},
+    io::{self, Write},
+    sync::Mutex,
+};
+
+use bdk_chain::{
+    bitcoin::{constants::genesis_block, Address, Network, OutPoint, ScriptBuf, Txid},
+    indexed_tx_graph::{self, IndexedTxGraph},
+    keychain,
+    local_chain::{self, LocalChain},
+    Append, ConfirmationTimeHeightAnchor,
+};
+
+use bdk_esplora::{esplora_client, EsploraExt};
+
+use example_cli::{
+    anyhow::{self, Context},
+    clap::{self, Parser, Subcommand},
+    Keychain,
+};
+
+const DB_MAGIC: &[u8] = b"bdk_example_esplora";
+const DB_PATH: &str = ".bdk_esplora_example.db";
+
+type ChangeSet = (
+    local_chain::ChangeSet,
+    indexed_tx_graph::ChangeSet<ConfirmationTimeHeightAnchor, keychain::ChangeSet<Keychain>>,
+);
+
+#[derive(Subcommand, Debug, Clone)]
+enum EsploraCommands {
+    /// Scans the addresses in the wallet using the esplora API.
+    Scan {
+        /// When a gap this large has been found for a keychain, it will stop.
+        #[clap(long, default_value = "5")]
+        stop_gap: usize,
+        #[clap(flatten)]
+        scan_options: ScanOptions,
+        #[clap(flatten)]
+        esplora_args: EsploraArgs,
+    },
+    /// Scan for particular addresses and unconfirmed transactions using the esplora API.
+    Sync {
+        /// Scan all the unused addresses.
+        #[clap(long)]
+        unused_spks: bool,
+        /// Scan every address that you have derived.
+        #[clap(long)]
+        all_spks: bool,
+        /// Scan unspent outpoints for spends or changes to confirmation status of residing tx.
+        #[clap(long)]
+        utxos: bool,
+        /// Scan unconfirmed transactions for updates.
+        #[clap(long)]
+        unconfirmed: bool,
+        #[clap(flatten)]
+        scan_options: ScanOptions,
+        #[clap(flatten)]
+        esplora_args: EsploraArgs,
+    },
+}
+impl EsploraCommands {
+    fn esplora_args(&self) -> EsploraArgs {
+        match self {
+            EsploraCommands::Scan { esplora_args, .. } => esplora_args.clone(),
+            EsploraCommands::Sync { esplora_args, .. } => esplora_args.clone(),
+        }
+    }
+}
+
+#[derive(clap::Args, Debug, Clone)]
+pub struct EsploraArgs {
+    /// The esplora url endpoint to connect to e.g. `<https://blockstream.info/api>`
+    /// If not provided it'll be set to a default for the network provided
+    esplora_url: Option<String>,
+}
+
+impl EsploraArgs {
+    pub fn client(&self, network: Network) -> anyhow::Result<esplora_client::BlockingClient> {
+        let esplora_url = self.esplora_url.as_deref().unwrap_or(match network {
+            Network::Bitcoin => "https://blockstream.info/api",
+            Network::Testnet => "https://blockstream.info/testnet/api",
+            Network::Regtest => "http://localhost:3002",
+            Network::Signet => "https://mempool.space/signet/api",
+            _ => panic!("unsupported network"),
+        });
+
+        let client = esplora_client::Builder::new(esplora_url).build_blocking()?;
+        Ok(client)
+    }
+}
+
+#[derive(Parser, Debug, Clone, PartialEq)]
+pub struct ScanOptions {
+    /// Max number of concurrent esplora server requests.
+    #[clap(long, default_value = "1")]
+    pub parallel_requests: usize,
+}
+
+fn main() -> anyhow::Result<()> {
+    let example_cli::Init {
+        args,
+        keymap,
+        index,
+        db,
+        init_changeset,
+    } = example_cli::init::<EsploraCommands, EsploraArgs, ChangeSet>(DB_MAGIC, DB_PATH)?;
+
+    let genesis_hash = genesis_block(args.network).block_hash();
+
+    let (init_chain_changeset, init_indexed_tx_graph_changeset) = init_changeset;
+
+    // Construct `IndexedTxGraph` and `LocalChain` with our initial changeset. They are wrapped in
+    // `Mutex` to display how they can be used in a multithreaded context. Technically the mutexes
+    // aren't strictly needed here.
+    let graph = Mutex::new({
+        let mut graph = IndexedTxGraph::new(index);
+        graph.apply_changeset(init_indexed_tx_graph_changeset);
+        graph
+    });
+    let chain = Mutex::new({
+        let (mut chain, _) = LocalChain::from_genesis_hash(genesis_hash);
+        chain.apply_changeset(&init_chain_changeset)?;
+        chain
+    });
+
+    let esplora_cmd = match &args.command {
+        // These are commands that are handled by this example (sync, scan).
+        example_cli::Commands::ChainSpecific(esplora_cmd) => esplora_cmd,
+        // These are general commands handled by example_cli. Execute the cmd and return.
+        general_cmd => {
+            return example_cli::handle_commands(
+                &graph,
+                &db,
+                &chain,
+                &keymap,
+                args.network,
+                |esplora_args, tx| {
+                    let client = esplora_args.client(args.network)?;
+                    client
+                        .broadcast(tx)
+                        .map(|_| ())
+                        .map_err(anyhow::Error::from)
+                },
+                general_cmd.clone(),
+            );
+        }
+    };
+
+    let client = esplora_cmd.esplora_args().client(args.network)?;
+    // Prepare the `IndexedTxGraph` update based on whether we are scanning or syncing.
+    // Scanning: We are iterating through spks of all keychains and scanning for transactions for
+    //   each spk. We start with the lowest derivation index spk and stop scanning after `stop_gap`
+    //   number of consecutive spks have no transaction history. A Scan is done in situations of
+    //   wallet restoration. It is a special case. Applications should use "sync" style updates
+    //   after an initial scan.
+    // Syncing: We only check for specified spks, utxos and txids to update their confirmation
+    //   status or fetch missing transactions.
+    let indexed_tx_graph_changeset = match &esplora_cmd {
+        EsploraCommands::Scan {
+            stop_gap,
+            scan_options,
+            ..
+        } => {
+            let keychain_spks = graph
+                .lock()
+                .expect("mutex must not be poisoned")
+                .index
+                .all_unbounded_spk_iters()
+                .into_iter()
+                // This `map` is purely for logging.
+                .map(|(keychain, iter)| {
+                    let mut first = true;
+                    let spk_iter = iter.inspect(move |(i, _)| {
+                        if first {
+                            eprint!("\nscanning {}: ", keychain);
+                            first = false;
+                        }
+                        eprint!("{} ", i);
+                        // Flush early to ensure we print at every iteration.
+                        let _ = io::stderr().flush();
+                    });
+                    (keychain, spk_iter)
+                })
+                .collect::<BTreeMap<_, _>>();
+
+            // The client scans keychain spks for transaction histories, stopping after `stop_gap`
+            // is reached. It returns a `TxGraph` update (`graph_update`) and a structure that
+            // represents the last active spk derivation indices of keychains
+            // (`keychain_indices_update`).
+            let (graph_update, last_active_indices) = client
+                .full_scan(keychain_spks, *stop_gap, scan_options.parallel_requests)
+                .context("scanning for transactions")?;
+
+            let mut graph = graph.lock().expect("mutex must not be poisoned");
+            // Because we did a stop gap based scan we are likely to have some updates to our
+            // deriviation indices. Usually before a scan you are on a fresh wallet with no
+            // addresses derived so we need to derive up to last active addresses the scan found
+            // before adding the transactions.
+            let (_, index_changeset) = graph.index.reveal_to_target_multi(&last_active_indices);
+            let mut indexed_tx_graph_changeset = graph.apply_update(graph_update);
+            indexed_tx_graph_changeset.append(index_changeset.into());
+            indexed_tx_graph_changeset
+        }
+        EsploraCommands::Sync {
+            mut unused_spks,
+            all_spks,
+            mut utxos,
+            mut unconfirmed,
+            scan_options,
+            ..
+        } => {
+            if !(*all_spks || unused_spks || utxos || unconfirmed) {
+                // If nothing is specifically selected, we select everything (except all spks).
+                unused_spks = true;
+                unconfirmed = true;
+                utxos = true;
+            } else if *all_spks {
+                // If all spks is selected, we don't need to also select unused spks (as unused spks
+                // is a subset of all spks).
+                unused_spks = false;
+            }
+
+            // Spks, outpoints and txids we want updates on will be accumulated here.
+            let mut spks: Box<dyn Iterator<Item = ScriptBuf>> = Box::new(core::iter::empty());
+            let mut outpoints: Box<dyn Iterator<Item = OutPoint>> = Box::new(core::iter::empty());
+            let mut txids: Box<dyn Iterator<Item = Txid>> = Box::new(core::iter::empty());
+
+            // Get a short lock on the structures to get spks, utxos, and txs that we are interested
+            // in.
+            {
+                let graph = graph.lock().unwrap();
+                let chain = chain.lock().unwrap();
+                let chain_tip = chain.tip().block_id();
+
+                if *all_spks {
+                    let all_spks = graph
+                        .index
+                        .revealed_spks()
+                        .map(|(k, i, spk)| (k, i, spk.to_owned()))
+                        .collect::<Vec<_>>();
+                    spks = Box::new(spks.chain(all_spks.into_iter().map(|(k, i, spk)| {
+                        eprintln!("scanning {}:{}", k, i);
+                        // Flush early to ensure we print at every iteration.
+                        let _ = io::stderr().flush();
+                        spk
+                    })));
+                }
+                if unused_spks {
+                    let unused_spks = graph
+                        .index
+                        .unused_spks()
+                        .map(|(k, i, spk)| (k, i, spk.to_owned()))
+                        .collect::<Vec<_>>();
+                    spks = Box::new(spks.chain(unused_spks.into_iter().map(|(k, i, spk)| {
+                        eprintln!(
+                            "Checking if address {} {}:{} has been used",
+                            Address::from_script(&spk, args.network).unwrap(),
+                            k,
+                            i,
+                        );
+                        // Flush early to ensure we print at every iteration.
+                        let _ = io::stderr().flush();
+                        spk
+                    })));
+                }
+                if utxos {
+                    // We want to search for whether the UTXO is spent, and spent by which
+                    // transaction. We provide the outpoint of the UTXO to
+                    // `EsploraExt::update_tx_graph_without_keychain`.
+                    let init_outpoints = graph.index.outpoints().iter().cloned();
+                    let utxos = graph
+                        .graph()
+                        .filter_chain_unspents(&*chain, chain_tip, init_outpoints)
+                        .map(|(_, utxo)| utxo)
+                        .collect::<Vec<_>>();
+                    outpoints = Box::new(
+                        utxos
+                            .into_iter()
+                            .inspect(|utxo| {
+                                eprintln!(
+                                    "Checking if outpoint {} (value: {}) has been spent",
+                                    utxo.outpoint, utxo.txout.value
+                                );
+                                // Flush early to ensure we print at every iteration.
+                                let _ = io::stderr().flush();
+                            })
+                            .map(|utxo| utxo.outpoint),
+                    );
+                };
+                if unconfirmed {
+                    // We want to search for whether the unconfirmed transaction is now confirmed.
+                    // We provide the unconfirmed txids to
+                    // `EsploraExt::update_tx_graph_without_keychain`.
+                    let unconfirmed_txids = graph
+                        .graph()
+                        .list_chain_txs(&*chain, chain_tip)
+                        .filter(|canonical_tx| !canonical_tx.chain_position.is_confirmed())
+                        .map(|canonical_tx| canonical_tx.tx_node.txid)
+                        .collect::<Vec<Txid>>();
+                    txids = Box::new(unconfirmed_txids.into_iter().inspect(|txid| {
+                        eprintln!("Checking if {} is confirmed yet", txid);
+                        // Flush early to ensure we print at every iteration.
+                        let _ = io::stderr().flush();
+                    }));
+                }
+            }
+
+            let graph_update =
+                client.sync(spks, txids, outpoints, scan_options.parallel_requests)?;
+
+            graph.lock().unwrap().apply_update(graph_update)
+        }
+    };
+
+    println!();
+
+    // Now that we're done updating the `IndexedTxGraph`, it's time to update the `LocalChain`! We
+    // want the `LocalChain` to have data about all the anchors in the `TxGraph` - for this reason,
+    // we want retrieve the blocks at the heights of the newly added anchors that are missing from
+    // our view of the chain.
+    let (missing_block_heights, tip) = {
+        let chain = &*chain.lock().unwrap();
+        let missing_block_heights = indexed_tx_graph_changeset
+            .graph
+            .missing_heights_from(chain)
+            .collect::<BTreeSet<_>>();
+        let tip = chain.tip();
+        (missing_block_heights, tip)
+    };
+
+    println!("prev tip: {}", tip.height());
+    println!("missing block heights: {:?}", missing_block_heights);
+
+    // Here, we actually fetch the missing blocks and create a `local_chain::Update`.
+    let chain_changeset = {
+        let chain_update = client
+            .update_local_chain(tip, missing_block_heights)
+            .context("scanning for blocks")?;
+        println!("new tip: {}", chain_update.tip.height());
+        chain.lock().unwrap().apply_update(chain_update)?
+    };
+
+    // We persist the changes
+    let mut db = db.lock().unwrap();
+    db.stage((chain_changeset, indexed_tx_graph_changeset));
+    db.commit()?;
+    Ok(())
+}

example-crates/wallet_electrum/Cargo.toml

@@ -7,3 +7,4 @@ edition = "2021"
 bdk = { path = "../../crates/bdk" }
 bdk_electrum = { path = "../../crates/electrum" }
 bdk_file_store = { path = "../../crates/file_store" }
+anyhow = "1"

example-crates/wallet_electrum/src/main.rs

@@ -7,26 +7,29 @@ use std::io::Write;
 use std::str::FromStr;
 
 use bdk::bitcoin::Address;
+use bdk::wallet::Update;
 use bdk::SignOptions;
 use bdk::{bitcoin::Network, Wallet};
-use bdk_electrum::electrum_client::{self, ElectrumApi};
-use bdk_electrum::ElectrumExt;
+use bdk_electrum::{
+    electrum_client::{self, ElectrumApi},
+    ElectrumExt, ElectrumUpdate,
+};
 use bdk_file_store::Store;
 
-fn main() -> Result<(), Box<dyn std::error::Error>> {
+fn main() -> Result<(), anyhow::Error> {
     let db_path = std::env::temp_dir().join("bdk-electrum-example");
-    let db = Store::<bdk::wallet::ChangeSet>::new_from_path(DB_MAGIC.as_bytes(), db_path)?;
-    let external_descriptor = "wpkh(tprv8ZgxMBicQKsPdy6LMhUtFHAgpocR8GC6QmwMSFpZs7h6Eziw3SpThFfczTDh5rW2krkqffa11UpX3XkeTTB2FvzZKWXqPY54Y6Rq4AQ5R8L/84'/0'/0'/0/*)";
-    let internal_descriptor = "wpkh(tprv8ZgxMBicQKsPdy6LMhUtFHAgpocR8GC6QmwMSFpZs7h6Eziw3SpThFfczTDh5rW2krkqffa11UpX3XkeTTB2FvzZKWXqPY54Y6Rq4AQ5R8L/84'/0'/0'/1/*)";
+    let db = Store::<bdk::wallet::ChangeSet>::open_or_create_new(DB_MAGIC.as_bytes(), db_path)?;
+    let external_descriptor = "wpkh(tprv8ZgxMBicQKsPdy6LMhUtFHAgpocR8GC6QmwMSFpZs7h6Eziw3SpThFfczTDh5rW2krkqffa11UpX3XkeTTB2FvzZKWXqPY54Y6Rq4AQ5R8L/84'/1'/0'/0/*)";
+    let internal_descriptor = "wpkh(tprv8ZgxMBicQKsPdy6LMhUtFHAgpocR8GC6QmwMSFpZs7h6Eziw3SpThFfczTDh5rW2krkqffa11UpX3XkeTTB2FvzZKWXqPY54Y6Rq4AQ5R8L/84'/1'/0'/1/*)";
 
-    let mut wallet = Wallet::new(
+    let mut wallet = Wallet::new_or_load(
         external_descriptor,
         Some(internal_descriptor),
         db,
         Network::Testnet,
     )?;
 
-    let address = wallet.get_address(bdk::wallet::AddressIndex::New);
+    let address = wallet.try_get_address(bdk::wallet::AddressIndex::New)?;
     println!("Generated Address: {}", address);
 
     let balance = wallet.get_balance();
@@ -35,9 +38,9 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     print!("Syncing...");
     let client = electrum_client::Client::new("ssl://electrum.blockstream.info:60002")?;
 
-    let local_chain = wallet.checkpoints();
+    let prev_tip = wallet.latest_checkpoint();
     let keychain_spks = wallet
-        .spks_of_all_keychains()
+        .all_unbounded_spk_iters()
         .into_iter()
         .map(|(k, k_spks)| {
             let mut once = Some(());
@@ -52,15 +55,25 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
         })
         .collect();
 
-    let electrum_update =
-        client.scan(local_chain, keychain_spks, None, None, STOP_GAP, BATCH_SIZE)?;
+    let (
+        ElectrumUpdate {
+            chain_update,
+            relevant_txids,
+        },
+        keychain_update,
+    ) = client.full_scan(prev_tip, keychain_spks, STOP_GAP, BATCH_SIZE)?;
 
     println!();
 
-    let missing = electrum_update.missing_full_txs(wallet.as_ref());
-    let update = electrum_update.finalize_as_confirmation_time(&client, None, missing)?;
+    let missing = relevant_txids.missing_full_txs(wallet.as_ref());
+    let graph_update = relevant_txids.into_confirmation_time_tx_graph(&client, None, missing)?;
 
-    wallet.apply_update(update)?;
+    let wallet_update = Update {
+        last_active_indices: keychain_update,
+        graph: graph_update,
+        chain: Some(chain_update),
+    };
+    wallet.apply_update(wallet_update)?;
     wallet.commit()?;
 
     let balance = wallet.get_balance();
@@ -74,14 +87,15 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
         std::process::exit(0);
     }
 
-    let faucet_address = Address::from_str("mkHS9ne12qx9pS9VojpwU5xtRd4T7X7ZUt")?;
+    let faucet_address = Address::from_str("mkHS9ne12qx9pS9VojpwU5xtRd4T7X7ZUt")?
+        .require_network(Network::Testnet)?;
 
     let mut tx_builder = wallet.build_tx();
     tx_builder
         .add_recipient(faucet_address.script_pubkey(), SEND_AMOUNT)
         .enable_rbf();
 
-    let (mut psbt, _) = tx_builder.finish()?;
+    let mut psbt = tx_builder.finish()?;
     let finalized = wallet.sign(&mut psbt, SignOptions::default())?;
     assert!(finalized);
 

example-crates/wallet_esplora_async/Cargo.toml

@@ -10,3 +10,4 @@ bdk = { path = "../../crates/bdk" }
 bdk_esplora = { path = "../../crates/esplora", features = ["async-https"] }
 bdk_file_store = { path = "../../crates/file_store" }
 tokio = { version = "1", features = ["rt", "rt-multi-thread", "macros"] }
+anyhow = "1"

example-crates/wallet_esplora_async/src/main.rs

@@ -2,7 +2,7 @@ use std::{io::Write, str::FromStr};
 
 use bdk::{
     bitcoin::{Address, Network},
-    wallet::AddressIndex,
+    wallet::{AddressIndex, Update},
     SignOptions, Wallet,
 };
 use bdk_esplora::{esplora_client, EsploraAsyncExt};
@@ -14,20 +14,20 @@ const STOP_GAP: usize = 50;
 const PARALLEL_REQUESTS: usize = 5;
 
 #[tokio::main]
-async fn main() -> Result<(), Box<dyn std::error::Error>> {
+async fn main() -> Result<(), anyhow::Error> {
     let db_path = std::env::temp_dir().join("bdk-esplora-async-example");
-    let db = Store::<bdk::wallet::ChangeSet>::new_from_path(DB_MAGIC.as_bytes(), db_path)?;
-    let external_descriptor = "wpkh(tprv8ZgxMBicQKsPdy6LMhUtFHAgpocR8GC6QmwMSFpZs7h6Eziw3SpThFfczTDh5rW2krkqffa11UpX3XkeTTB2FvzZKWXqPY54Y6Rq4AQ5R8L/84'/0'/0'/0/*)";
-    let internal_descriptor = "wpkh(tprv8ZgxMBicQKsPdy6LMhUtFHAgpocR8GC6QmwMSFpZs7h6Eziw3SpThFfczTDh5rW2krkqffa11UpX3XkeTTB2FvzZKWXqPY54Y6Rq4AQ5R8L/84'/0'/0'/1/*)";
+    let db = Store::<bdk::wallet::ChangeSet>::open_or_create_new(DB_MAGIC.as_bytes(), db_path)?;
+    let external_descriptor = "wpkh(tprv8ZgxMBicQKsPdy6LMhUtFHAgpocR8GC6QmwMSFpZs7h6Eziw3SpThFfczTDh5rW2krkqffa11UpX3XkeTTB2FvzZKWXqPY54Y6Rq4AQ5R8L/84'/1'/0'/0/*)";
+    let internal_descriptor = "wpkh(tprv8ZgxMBicQKsPdy6LMhUtFHAgpocR8GC6QmwMSFpZs7h6Eziw3SpThFfczTDh5rW2krkqffa11UpX3XkeTTB2FvzZKWXqPY54Y6Rq4AQ5R8L/84'/1'/0'/1/*)";
 
-    let mut wallet = Wallet::new(
+    let mut wallet = Wallet::new_or_load(
         external_descriptor,
         Some(internal_descriptor),
         db,
         Network::Testnet,
     )?;
 
-    let address = wallet.get_address(AddressIndex::New);
+    let address = wallet.try_get_address(AddressIndex::New)?;
     println!("Generated Address: {}", address);
 
     let balance = wallet.get_balance();
@@ -37,9 +37,9 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     let client =
         esplora_client::Builder::new("https://blockstream.info/testnet/api").build_async()?;
 
-    let local_chain = wallet.checkpoints();
+    let prev_tip = wallet.latest_checkpoint();
     let keychain_spks = wallet
-        .spks_of_all_keychains()
+        .all_unbounded_spk_iters()
         .into_iter()
         .map(|(k, k_spks)| {
             let mut once = Some(());
@@ -53,19 +53,19 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
             (k, k_spks)
         })
         .collect();
-    let update = client
-        .scan(
-            local_chain,
-            keychain_spks,
-            [],
-            [],
-            STOP_GAP,
-            PARALLEL_REQUESTS,
-        )
+    let (update_graph, last_active_indices) = client
+        .full_scan(keychain_spks, STOP_GAP, PARALLEL_REQUESTS)
         .await?;
-    println!();
+    let missing_heights = update_graph.missing_heights(wallet.local_chain());
+    let chain_update = client.update_local_chain(prev_tip, missing_heights).await?;
+    let update = Update {
+        last_active_indices,
+        graph: update_graph,
+        chain: Some(chain_update),
+    };
     wallet.apply_update(update)?;
     wallet.commit()?;
+    println!();
 
     let balance = wallet.get_balance();
     println!("Wallet balance after syncing: {} sats", balance.total());
@@ -78,14 +78,15 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
         std::process::exit(0);
     }
 
-    let faucet_address = Address::from_str("mkHS9ne12qx9pS9VojpwU5xtRd4T7X7ZUt")?;
+    let faucet_address = Address::from_str("mkHS9ne12qx9pS9VojpwU5xtRd4T7X7ZUt")?
+        .require_network(Network::Testnet)?;
 
     let mut tx_builder = wallet.build_tx();
     tx_builder
         .add_recipient(faucet_address.script_pubkey(), SEND_AMOUNT)
         .enable_rbf();
 
-    let (mut psbt, _) = tx_builder.finish()?;
+    let mut psbt = tx_builder.finish()?;
     let finalized = wallet.sign(&mut psbt, SignOptions::default())?;
     assert!(finalized);
 

example-crates/wallet_esplora_blocking/Cargo.toml

@@ -1,5 +1,5 @@
 [package]
-name = "wallet_esplora"
+name = "wallet_esplora_blocking"
 version = "0.2.0"
 edition = "2021"
 publish = false
@@ -10,3 +10,4 @@ publish = false
 bdk = { path = "../../crates/bdk" }
 bdk_esplora = { path = "../../crates/esplora", features = ["blocking"] }
 bdk_file_store = { path = "../../crates/file_store" }
+anyhow = "1"

example-crates/wallet_esplora_blocking/src/main.rs

@@ -1,32 +1,32 @@
 const DB_MAGIC: &str = "bdk_wallet_esplora_example";
-const SEND_AMOUNT: u64 = 5000;
-const STOP_GAP: usize = 50;
-const PARALLEL_REQUESTS: usize = 5;
+const SEND_AMOUNT: u64 = 1000;
+const STOP_GAP: usize = 5;
+const PARALLEL_REQUESTS: usize = 1;
 
 use std::{io::Write, str::FromStr};
 
 use bdk::{
     bitcoin::{Address, Network},
-    wallet::AddressIndex,
+    wallet::{AddressIndex, Update},
     SignOptions, Wallet,
 };
 use bdk_esplora::{esplora_client, EsploraExt};
 use bdk_file_store::Store;
 
-fn main() -> Result<(), Box<dyn std::error::Error>> {
+fn main() -> Result<(), anyhow::Error> {
     let db_path = std::env::temp_dir().join("bdk-esplora-example");
-    let db = Store::<bdk::wallet::ChangeSet>::new_from_path(DB_MAGIC.as_bytes(), db_path)?;
-    let external_descriptor = "wpkh(tprv8ZgxMBicQKsPdy6LMhUtFHAgpocR8GC6QmwMSFpZs7h6Eziw3SpThFfczTDh5rW2krkqffa11UpX3XkeTTB2FvzZKWXqPY54Y6Rq4AQ5R8L/84'/0'/0'/0/*)";
-    let internal_descriptor = "wpkh(tprv8ZgxMBicQKsPdy6LMhUtFHAgpocR8GC6QmwMSFpZs7h6Eziw3SpThFfczTDh5rW2krkqffa11UpX3XkeTTB2FvzZKWXqPY54Y6Rq4AQ5R8L/84'/0'/0'/1/*)";
+    let db = Store::<bdk::wallet::ChangeSet>::open_or_create_new(DB_MAGIC.as_bytes(), db_path)?;
+    let external_descriptor = "wpkh(tprv8ZgxMBicQKsPdy6LMhUtFHAgpocR8GC6QmwMSFpZs7h6Eziw3SpThFfczTDh5rW2krkqffa11UpX3XkeTTB2FvzZKWXqPY54Y6Rq4AQ5R8L/84'/1'/0'/0/*)";
+    let internal_descriptor = "wpkh(tprv8ZgxMBicQKsPdy6LMhUtFHAgpocR8GC6QmwMSFpZs7h6Eziw3SpThFfczTDh5rW2krkqffa11UpX3XkeTTB2FvzZKWXqPY54Y6Rq4AQ5R8L/84'/1'/0'/1/*)";
 
-    let mut wallet = Wallet::new(
+    let mut wallet = Wallet::new_or_load(
         external_descriptor,
         Some(internal_descriptor),
         db,
         Network::Testnet,
     )?;
 
-    let address = wallet.get_address(AddressIndex::New);
+    let address = wallet.try_get_address(AddressIndex::New)?;
     println!("Generated Address: {}", address);
 
     let balance = wallet.get_balance();
@@ -36,9 +36,9 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     let client =
         esplora_client::Builder::new("https://blockstream.info/testnet/api").build_blocking()?;
 
-    let local_chain = wallet.checkpoints();
+    let prev_tip = wallet.latest_checkpoint();
     let keychain_spks = wallet
-        .spks_of_all_keychains()
+        .all_unbounded_spk_iters()
         .into_iter()
         .map(|(k, k_spks)| {
             let mut once = Some(());
@@ -52,17 +52,20 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
             (k, k_spks)
         })
         .collect();
-    let update = client.scan(
-        local_chain,
-        keychain_spks,
-        None,
-        None,
-        STOP_GAP,
-        PARALLEL_REQUESTS,
-    )?;
-    println!();
+
+    let (update_graph, last_active_indices) =
+        client.full_scan(keychain_spks, STOP_GAP, PARALLEL_REQUESTS)?;
+    let missing_heights = update_graph.missing_heights(wallet.local_chain());
+    let chain_update = client.update_local_chain(prev_tip, missing_heights)?;
+    let update = Update {
+        last_active_indices,
+        graph: update_graph,
+        chain: Some(chain_update),
+    };
+
     wallet.apply_update(update)?;
     wallet.commit()?;
+    println!();
 
     let balance = wallet.get_balance();
     println!("Wallet balance after syncing: {} sats", balance.total());
@@ -75,14 +78,15 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
         std::process::exit(0);
     }
 
-    let faucet_address = Address::from_str("mkHS9ne12qx9pS9VojpwU5xtRd4T7X7ZUt")?;
+    let faucet_address = Address::from_str("mkHS9ne12qx9pS9VojpwU5xtRd4T7X7ZUt")?
+        .require_network(Network::Testnet)?;
 
     let mut tx_builder = wallet.build_tx();
     tx_builder
         .add_recipient(faucet_address.script_pubkey(), SEND_AMOUNT)
         .enable_rbf();
 
-    let (mut psbt, _) = tx_builder.finish()?;
+    let mut psbt = tx_builder.finish()?;
     let finalized = wallet.sign(&mut psbt, SignOptions::default())?;
     assert!(finalized);