Add wallet persistence

Cargo.toml

@@ -37,6 +37,7 @@ js-sys = "0.3"
 [features]
 default = ["std"]
 std = []
+file-store = [ "std", "bdk_chain/file_store"]
 compiler = ["miniscript/compiler"]
 all-keys = ["keys-bip39"]
 keys-bip39 = ["bip39"]

src/descriptor/mod.rs

@@ -39,9 +39,7 @@ pub mod checksum;
 pub mod dsl;
 pub mod error;
 pub mod policy;
-mod spk_iter;
 pub mod template;
-pub use spk_iter::SpkIter;
 
 pub use self::checksum::calc_checksum;
 use self::checksum::calc_checksum_bytes;

src/descriptor/spk_iter.rs

@@ -1,63 +0,0 @@
-use bitcoin::{
-    secp256k1::{Secp256k1, VerifyOnly},
-    Script,
-};
-use miniscript::{Descriptor, DescriptorPublicKey};
-
-/// An iterator over a descriptor's script pubkeys.
-///
-// TODO: put this into miniscript
-#[derive(Clone, Debug)]
-pub struct SpkIter {
-    descriptor: Descriptor<DescriptorPublicKey>,
-    index: usize,
-    secp: Secp256k1<VerifyOnly>,
-    end: usize,
-}
-
-impl SpkIter {
-    /// Creates a new script pubkey iterator starting at 0 from a descriptor
-    pub fn new(descriptor: Descriptor<DescriptorPublicKey>) -> Self {
-        let secp = Secp256k1::verification_only();
-        let end = if descriptor.has_wildcard() {
-            // Because we only iterate over non-hardened indexes there are 2^31 values
-            (1 << 31) - 1
-        } else {
-            0
-        };
-
-        Self {
-            descriptor,
-            index: 0,
-            secp,
-            end,
-        }
-    }
-}
-
-impl Iterator for SpkIter {
-    type Item = (u32, Script);
-
-    fn nth(&mut self, n: usize) -> Option<Self::Item> {
-        self.index = self.index.saturating_add(n);
-        self.next()
-    }
-
-    fn next(&mut self) -> Option<Self::Item> {
-        let index = self.index;
-        if index > self.end {
-            return None;
-        }
-
-        let script = self
-            .descriptor
-            .at_derivation_index(self.index as u32)
-            .derived_descriptor(&self.secp)
-            .expect("the descritpor cannot need hardened derivation")
-            .script_pubkey();
-
-        self.index += 1;
-
-        Some((index as u32, script))
-    }
-}

src/wallet/export.rs

@@ -117,8 +117,8 @@ impl FullyNodedExport {
     ///
     /// If the database is empty or `include_blockheight` is false, the `blockheight` field
     /// returned will be `0`.
-    pub fn export_wallet(
+    pub fn export_wallet<D>(
-        wallet: &Wallet,
+        wallet: &Wallet<D>,
         label: &str,
         include_blockheight: bool,
     ) -> Result<Self, &'static str> {
@@ -231,8 +231,8 @@ mod test {
         descriptor: &str,
         change_descriptor: Option<&str>,
         network: Network,
-    ) -> Wallet {
+    ) -> Wallet<()> {
-        let mut wallet = Wallet::new(descriptor, change_descriptor, network).unwrap();
+        let mut wallet = Wallet::new(descriptor, change_descriptor, (), network).unwrap();
         let transaction = Transaction {
             input: vec![],
             output: vec![],

src/wallet/mod.rs

@@ -19,9 +19,12 @@ use alloc::{
     sync::Arc,
     vec::Vec,
 };
-use bdk_chain::{chain_graph, keychain::KeychainTracker, sparse_chain, BlockId, ConfirmationTime};
+use bdk_chain::{
+    chain_graph,
+    keychain::{KeychainChangeSet, KeychainScan, KeychainTracker},
+    sparse_chain, BlockId, ConfirmationTime, IntoOwned,
+};
 use bitcoin::secp256k1::Secp256k1;
-use core::convert::TryInto;
 use core::fmt;
 use core::ops::Deref;
 
@@ -46,6 +49,7 @@ pub(crate) mod utils;
 #[cfg(feature = "hardware-signer")]
 #[cfg_attr(docsrs, doc(cfg(feature = "hardware-signer")))]
 pub mod hardwaresigner;
+pub mod persist;
 
 pub use utils::IsDust;
 
@@ -58,7 +62,7 @@ use utils::{check_nsequence_rbf, After, Older, SecpCtx};
 use crate::descriptor::policy::BuildSatisfaction;
 use crate::descriptor::{
     calc_checksum, into_wallet_descriptor_checked, DerivedDescriptor, DescriptorMeta,
-    ExtendedDescriptor, ExtractPolicy, IntoWalletDescriptor, Policy, SpkIter, XKeyUtils,
+    ExtendedDescriptor, ExtractPolicy, IntoWalletDescriptor, Policy, XKeyUtils,
 };
 use crate::error::{Error, MiniscriptPsbtError};
 use crate::psbt::PsbtUtils;
@@ -80,16 +84,23 @@ const COINBASE_MATURITY: u32 = 100;
 /// [`Database`]: crate::database::Database
 /// [`signer`]: crate::signer
 #[derive(Debug)]
-pub struct Wallet {
+pub struct Wallet<D = ()> {
     signers: Arc<SignersContainer>,
     change_signers: Arc<SignersContainer>,
     keychain_tracker: KeychainTracker<KeychainKind, ConfirmationTime>,
-
+    persist: persist::Persist<D>,
     network: Network,
-
     secp: SecpCtx,
 }
 
+/// The update to a [`Wallet`] used in [Wallet::apply_update]. This is usually returned from blockchain data sources.
+/// The type parameter `T` indicates the kind of transaction contained in the update. It's usually a [`bitcoin::Transaction`].
+pub type Update<T> = KeychainScan<KeychainKind, ConfirmationTime, T>;
+/// Error indicating that something was wrong with an [`Update<T>`].
+pub type UpdateError = chain_graph::UpdateError<ConfirmationTime>;
+/// The changeset produced internally by applying an update
+pub(crate) type ChangeSet = KeychainChangeSet<KeychainKind, ConfirmationTime, Transaction>;
+
 /// The address index selection strategy to use to derived an address from the wallet's external
 /// descriptor. See [`Wallet::get_address`]. If you're unsure which one to use use `WalletIndex::New`.
 #[derive(Debug)]
@@ -139,18 +150,62 @@ impl fmt::Display for AddressInfo {
 }
 
 impl Wallet {
-    /// Create a wallet.
+    /// Creates a wallet that does not persist data.
-    ///
+    pub fn new_no_persist<E: IntoWalletDescriptor>(
-    /// The only way this can fail is if the descriptors passed in do not match the checksums in `database`.
-    pub fn new<E: IntoWalletDescriptor>(
         descriptor: E,
         change_descriptor: Option<E>,
         network: Network,
-    ) -> Result<Self, Error> {
+    ) -> Result<Self, crate::descriptor::DescriptorError> {
+        Self::new(descriptor, change_descriptor, (), network).map_err(|e| match e {
+            NewError::Descriptor(e) => e,
+            NewError::Persist(_) => unreachable!("no persistence so it can't fail"),
+        })
+    }
+}
+
+#[derive(Debug)]
+/// Error returned from [`Wallet::new`]
+pub enum NewError<P> {
+    /// There was problem with the descriptors passed in
+    Descriptor(crate::descriptor::DescriptorError),
+    /// We were unable to load the wallet's data from the persistance backend
+    Persist(P),
+}
+
+impl<P> core::fmt::Display for NewError<P>
+where
+    P: core::fmt::Display,
+{
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            NewError::Descriptor(e) => e.fmt(f),
+            NewError::Persist(e) => {
+                write!(f, "failed to load wallet from persistance backend: {}", e)
+            }
+        }
+    }
+}
+
+#[cfg(feautre = "std")]
+impl<P: core::fmt::Display> std::error::Error for NewError<P> {}
+
+impl<D> Wallet<D> {
+    /// Create a wallet from a `descriptor` (and an optional `change_descriptor`) and load related
+    /// transaction data from `db`.
+    pub fn new<E: IntoWalletDescriptor>(
+        descriptor: E,
+        change_descriptor: Option<E>,
+        mut db: D,
+        network: Network,
+    ) -> Result<Self, NewError<D::LoadError>>
+    where
+        D: persist::Backend,
+    {
         let secp = Secp256k1::new();
 
         let mut keychain_tracker = KeychainTracker::default();
-        let (descriptor, keymap) = into_wallet_descriptor_checked(descriptor, &secp, network)?;
+        let (descriptor, keymap) = into_wallet_descriptor_checked(descriptor, &secp, network)
+            .map_err(NewError::Descriptor)?;
         keychain_tracker
             .txout_index
             .add_keychain(KeychainKind::External, descriptor.clone());
@@ -158,7 +213,8 @@ impl Wallet {
         let change_signers = match change_descriptor {
             Some(desc) => {
                 let (change_descriptor, change_keymap) =
-                    into_wallet_descriptor_checked(desc, &secp, network)?;
+                    into_wallet_descriptor_checked(desc, &secp, network)
+                        .map_err(NewError::Descriptor)?;
 
                 let change_signers = Arc::new(SignersContainer::build(
                     change_keymap,
@@ -175,10 +231,16 @@ impl Wallet {
             None => Arc::new(SignersContainer::new()),
         };
 
+        db.load_into_keychain_tracker(&mut keychain_tracker)
+            .map_err(NewError::Persist)?;
+
+        let persist = persist::Persist::new(db);
+
         Ok(Wallet {
             signers,
             change_signers,
             network,
+            persist,
             secp,
             keychain_tracker,
         })
@@ -194,55 +256,13 @@ impl Wallet {
         self.keychain_tracker.txout_index.keychains()
     }
 
-    // Return a newly derived address for the specified `keychain`.
-    fn get_new_address(&mut self, keychain: KeychainKind) -> AddressInfo {
-        let ((index, spk), _) = self.keychain_tracker.txout_index.reveal_next_spk(&keychain);
-        let address =
-            Address::from_script(&spk, self.network).expect("descriptor must have address form");
-
-        AddressInfo {
-            address,
-            index,
-            keychain,
-        }
-    }
-
-    // Return the the last previously derived address for `keychain` if it has not been used in a
-    // received transaction. Otherwise return a new address using [`Wallet::get_new_address`].
-    fn get_unused_address(&mut self, keychain: KeychainKind) -> AddressInfo {
-        let index = self.derivation_index(KeychainKind::External);
-
-        match index {
-            Some(index)
-                if !self
-                    .keychain_tracker
-                    .txout_index
-                    .is_used(&(keychain, index)) =>
-            {
-                self.peek_address(index, keychain)
-            }
-            _ => self.get_new_address(keychain),
-        }
-    }
-
-    // Return derived address for the descriptor of given [`KeychainKind`] at a specific index
-    fn peek_address(&self, index: u32, keychain: KeychainKind) -> AddressInfo {
-        let address = self
-            .get_descriptor_for_keychain(keychain)
-            .at_derivation_index(index)
-            .address(self.network)
-            .expect("descriptor must have address form");
-        AddressInfo {
-            index,
-            address,
-            keychain,
-        }
-    }
-
     /// Return a derived address using the external descriptor, see [`AddressIndex`] for
     /// available address index selection strategies. If none of the keys in the descriptor are derivable
     /// (i.e. does not end with /*) then the same address will always be returned for any [`AddressIndex`].
-    pub fn get_address(&mut self, address_index: AddressIndex) -> AddressInfo {
+    pub fn get_address(&mut self, address_index: AddressIndex) -> AddressInfo
+    where
+        D: persist::Backend,
+    {
         self._get_address(address_index, KeychainKind::External)
     }
 
@@ -253,19 +273,53 @@ impl Wallet {
     /// see [`AddressIndex`] for available address index selection strategies. If none of the keys
     /// in the descriptor are derivable (i.e. does not end with /*) then the same address will always
     /// be returned for any [`AddressIndex`].
-    pub fn get_internal_address(&mut self, address_index: AddressIndex) -> AddressInfo {
+    pub fn get_internal_address(&mut self, address_index: AddressIndex) -> AddressInfo
+    where
+        D: persist::Backend,
+    {
         self._get_address(address_index, KeychainKind::Internal)
     }
 
-    fn _get_address(&mut self, address_index: AddressIndex, keychain: KeychainKind) -> AddressInfo {
+    fn _get_address(&mut self, address_index: AddressIndex, keychain: KeychainKind) -> AddressInfo
-        // TODO: Fix this mess!
+    where
-        let _keychain = self.map_keychain(keychain);
+        D: persist::Backend,
-        let mut info = match address_index {
+    {
-            AddressIndex::New => self.get_new_address(_keychain),
+        let keychain = self.map_keychain(keychain);
-            AddressIndex::LastUnused => self.get_unused_address(_keychain),
+        let txout_index = &mut self.keychain_tracker.txout_index;
-            AddressIndex::Peek(index) => self.peek_address(index, _keychain),
+        let (index, spk) = match address_index {
+            AddressIndex::New => {
+                let ((index, spk), changeset) = txout_index.reveal_next_spk(&keychain);
+                let spk = spk.clone();
+
+                self.persist.stage(changeset.into());
+                self.persist.commit().expect("TODO");
+                (index, spk)
+            }
+            AddressIndex::LastUnused => {
+                let index = txout_index.last_revealed_index(&keychain);
+                match index {
+                    Some(index) if !txout_index.is_used(&(keychain, index)) => (
+                        index,
+                        txout_index
+                            .spk_at_index(&(keychain, index))
+                            .expect("must exist")
+                            .clone(),
+                    ),
+                    _ => return self._get_address(AddressIndex::New, keychain),
+                }
+            }
+            AddressIndex::Peek(index) => txout_index
+                .spks_of_keychain(&keychain)
+                .take(index as usize + 1)
+                .last()
+                .unwrap(),
+        };
+        let info = AddressInfo {
+            index,
+            address: Address::from_script(&spk, self.network)
+                .expect("descriptor must have address form"),
+            keychain,
         };
-        info.keychain = keychain;
         info
     }
 
@@ -295,7 +349,7 @@ impl Wallet {
             .collect()
     }
 
-    /// Iterate over all checkpoints.
+    /// Get all the checkpoints the wallet is currently storing indexed by height.
     pub fn checkpoints(&self) -> &BTreeMap<u32, BlockHash> {
         self.keychain_tracker.chain().checkpoints()
     }
@@ -305,13 +359,35 @@ impl Wallet {
         self.keychain_tracker.chain().latest_checkpoint()
     }
 
-    /// Create an iterator over all the script pubkeys starting at index 0 for a particular
+    /// Returns a iterators of all the script pubkeys for the `Internal` and External` variants in `KeychainKind`.
-    /// keychain.
+    ///
-    pub fn iter_all_script_pubkeys(&self, keychain: KeychainKind) -> SpkIter {
+    /// This is inteded to be used when doing a full scan of your addresses (e.g. after restoring
-        SpkIter::new(self.get_descriptor_for_keychain(keychain).clone())
+    /// from seed words). You pass the `BTreeMap` of iterators to a blockchain data source (e.g.
+    /// electrum server) which will go through each address until it reaches a *stop grap*.
+    ///
+    /// Note carefully that iterators go over **all** script pubkeys on the keychains (not what
+    /// script pubkeys the wallet is storing internally).
+    pub fn spks_of_all_keychains(
+        &self,
+    ) -> BTreeMap<KeychainKind, impl Iterator<Item = (u32, Script)> + Clone> {
+        self.keychain_tracker.txout_index.spks_of_all_keychains()
     }
 
-    /// Returns the `UTXO` owned by this wallet corresponding to `outpoint` if it exists in the
+    /// Gets an iterator over all the script pubkeys in a single keychain.
+    ///
+    /// See [`spks_of_all_keychains`] for more documentation
+    ///
+    /// [`spks_of_all_keychains`]: Self::spks_of_all_keychains
+    pub fn spks_of_keychain(
+        &self,
+        keychain: KeychainKind,
+    ) -> impl Iterator<Item = (u32, Script)> + Clone {
+        self.keychain_tracker
+            .txout_index
+            .spks_of_keychain(&keychain)
+    }
+
+    /// Returns the utxo owned by this wallet corresponding to `outpoint` if it exists in the
     /// wallet's database.
     pub fn get_utxo(&self, op: OutPoint) -> Option<LocalUtxo> {
         self.keychain_tracker
@@ -390,24 +466,46 @@ impl Wallet {
         })
     }
 
-    /// Add a new checkpoint to the wallet
+    /// Add a new checkpoint to the wallet's internal view of the chain.
+    /// This stages but does not [`commit`] the change.
+    ///
+    /// Returns whether anything changed with the insertion (e.g. `false` if checkpoint was already
+    /// there).
+    ///
+    /// [`commit`]: Self::commit
     pub fn insert_checkpoint(
         &mut self,
         block_id: BlockId,
     ) -> Result<bool, sparse_chain::InsertCheckpointError> {
-        Ok(!self
+        let changeset = self.keychain_tracker.insert_checkpoint(block_id)?;
-            .keychain_tracker
+        let changed = changeset.is_empty();
-            .insert_checkpoint(block_id)?
+        self.persist.stage(changeset);
-            .is_empty())
+        Ok(changed)
     }
 
-    /// Add a transaction to the wallet. Will only work if height <= latest checkpoint
+    /// Add a transaction to the wallet's internal view of the chain.
+    /// This stages but does not [`commit`] the change.
+    ///
+    /// There are a number reasons `tx` could be rejected with an `Err(_)`. The most important one
+    /// is that the transaction is at a height that is greater than [`latest_checkpoint`]. Therefore
+    /// you should use [`insert_checkpoint`] to insert new checkpoints before manually inserting new
+    /// transactions.
+    ///
+    /// Returns whether anything changed with the transaction insertion (e.g. `false` if the
+    /// transaction was already inserted at the same position).
+    ///
+    /// [`commit`]: Self::commit
+    /// [`latest_checkpoint`]: Self::latest_checkpoint
+    /// [`insert_checkpoint`]: Self::insert_checkpoint
     pub fn insert_tx(
         &mut self,
         tx: Transaction,
         position: ConfirmationTime,
     ) -> Result<bool, chain_graph::InsertTxError<ConfirmationTime>> {
-        Ok(!self.keychain_tracker.insert_tx(tx, position)?.is_empty())
+        let changeset = self.keychain_tracker.insert_tx(tx, position)?;
+        let changed = changeset.is_empty();
+        self.persist.stage(changeset);
+        Ok(changed)
     }
 
     #[deprecated(note = "use Wallet::transactions instead")]
@@ -426,17 +524,9 @@ impl Wallet {
         &self,
     ) -> impl DoubleEndedIterator<Item = (ConfirmationTime, &Transaction)> + '_ {
         self.keychain_tracker
-            .chain()
+            .chain_graph()
-            .txids()
+            .transactions_in_chain()
-            .map(move |&(pos, txid)| {
+            .map(|(pos, tx)| (*pos, tx))
-                (
-                    pos,
-                    self.keychain_tracker
-                        .graph()
-                        .get_tx(txid)
-                        .expect("must exist"),
-                )
-            })
     }
 
     /// Return the balance, separated into available, trusted-pending, untrusted-pending and immature
@@ -512,7 +602,7 @@ impl Wallet {
     /// # use bdk::{Wallet, KeychainKind};
     /// # use bdk::bitcoin::Network;
     /// # use bdk::database::MemoryDatabase;
-    /// let wallet = Wallet::new("wpkh(tprv8ZgxMBicQKsPe73PBRSmNbTfbcsZnwWhz5eVmhHpi31HW29Z7mc9B4cWGRQzopNUzZUT391DeDJxL2PefNunWyLgqCKRMDkU1s2s8bAfoSk/84'/0'/0'/0/*)", None, Network::Testnet, MemoryDatabase::new())?;
+    /// let wallet = Wallet::new_no_persist("wpkh(tprv8ZgxMBicQKsPe73PBRSmNbTfbcsZnwWhz5eVmhHpi31HW29Z7mc9B4cWGRQzopNUzZUT391DeDJxL2PefNunWyLgqCKRMDkU1s2s8bAfoSk/84'/0'/0'/0/*)", None, Network::Testnet, MemoryDatabase::new())?;
     /// for secret_key in wallet.get_signers(KeychainKind::External).signers().iter().filter_map(|s| s.descriptor_secret_key()) {
     ///     // secret_key: tprv8ZgxMBicQKsPe73PBRSmNbTfbcsZnwWhz5eVmhHpi31HW29Z7mc9B4cWGRQzopNUzZUT391DeDJxL2PefNunWyLgqCKRMDkU1s2s8bAfoSk/84'/0'/0'/0/*
     ///     println!("secret_key: {}", secret_key);
@@ -553,7 +643,7 @@ impl Wallet {
     /// ```
     ///
     /// [`TxBuilder`]: crate::TxBuilder
-    pub fn build_tx(&mut self) -> TxBuilder<'_, DefaultCoinSelectionAlgorithm, CreateTx> {
+    pub fn build_tx(&mut self) -> TxBuilder<'_, D, DefaultCoinSelectionAlgorithm, CreateTx> {
         TxBuilder {
             wallet: alloc::rc::Rc::new(core::cell::RefCell::new(self)),
             params: TxParams::default(),
@@ -566,7 +656,10 @@ impl Wallet {
         &mut self,
         coin_selection: Cs,
         params: TxParams,
-    ) -> Result<(psbt::PartiallySignedTransaction, TransactionDetails), Error> {
+    ) -> Result<(psbt::PartiallySignedTransaction, TransactionDetails), Error>
+    where
+        D: persist::Backend,
+    {
         let external_descriptor = self
             .keychain_tracker
             .txout_index
@@ -824,10 +917,20 @@ impl Wallet {
         // get drain script
         let drain_script = match params.drain_to {
             Some(ref drain_recipient) => drain_recipient.clone(),
-            None => self
+            None => {
-                .get_internal_address(AddressIndex::New)
+                let change_keychain = self.map_keychain(KeychainKind::Internal);
-                .address
+                let ((index, spk), changeset) = self
-                .script_pubkey(),
+                    .keychain_tracker
+                    .txout_index
+                    .next_unused_spk(&change_keychain);
+                let spk = spk.clone();
+                self.keychain_tracker
+                    .txout_index
+                    .mark_used(&change_keychain, index);
+                self.persist.stage(changeset.into());
+                self.persist.commit().expect("TODO");
+                spk
+            }
         };
 
         let coin_selection = coin_selection.coin_select(
@@ -961,7 +1064,7 @@ impl Wallet {
     pub fn build_fee_bump(
         &mut self,
         txid: Txid,
-    ) -> Result<TxBuilder<'_, DefaultCoinSelectionAlgorithm, BumpFee>, Error> {
+    ) -> Result<TxBuilder<'_, D, DefaultCoinSelectionAlgorithm, BumpFee>, Error> {
         let graph = self.keychain_tracker.graph();
         let txout_index = &self.keychain_tracker.txout_index;
         let tx_and_height = self.keychain_tracker.chain_graph().get_tx_in_chain(txid);
@@ -981,11 +1084,12 @@ impl Wallet {
             return Err(Error::IrreplaceableTransaction);
         }
 
-        let fee = graph
+        let fee = graph.calculate_fee(&tx).ok_or(Error::FeeRateUnavailable)?;
-            .calculate_fee(&tx)
+        if fee < 0 {
-            .ok_or(Error::FeeRateUnavailable)?
+            // It's available but it's wrong so let's say it's unavailable
-            .try_into()
+            return Err(Error::FeeRateUnavailable)?;
-            .map_err(|_| Error::FeeRateUnavailable)?;
+        }
+        let fee = fee as u64;
         let feerate = FeeRate::from_wu(fee, tx.weight());
 
         // remove the inputs from the tx and process them
@@ -1608,6 +1712,39 @@ impl Wallet {
             .1
             .to_string()
     }
+
+    /// Applies an update to the wallet and stages the changes (but does not [`commit`] them).
+    ///
+    /// Usually you create an `update` by interacting with some blockchain data source and inserting
+    /// transactions related to your wallet into it.
+    ///
+    /// [`commit`]: Self::commit
+    pub fn apply_udpate<Tx>(&mut self, update: Update<Tx>) -> Result<(), UpdateError>
+    where
+        D: persist::Backend,
+        Tx: IntoOwned<Transaction> + Clone,
+    {
+        let changeset = self.keychain_tracker.apply_update(update)?;
+        self.persist.stage(changeset);
+        Ok(())
+    }
+
+    /// Commits all curently [`staged`] changed to the persistence backend returning and error when this fails.
+    ///
+    /// [`staged`]: Self::staged
+    pub fn commit(&mut self) -> Result<(), D::WriteError>
+    where
+        D: persist::Backend,
+    {
+        self.persist.commit()
+    }
+
+    /// Returns the changes that will be staged with the next call to [`commit`].
+    ///
+    /// [`commit`]: Self::commit
+    pub fn staged(&self) -> &ChangeSet {
+        self.persist.staged()
+    }
 }
 
 /// Deterministically generate a unique name given the descriptors defining the wallet

src/wallet/persist.rs

@@ -0,0 +1,120 @@
+//! Persistence for changes made to a [`Wallet`].
+//!
+//! BDK's [`Wallet`] needs somewhere to persist changes it makes during operation.
+//! Operations like giving out a new address are crucial to persist so that next time the
+//! application is loaded it can find transactions related to that address.
+//!
+//! Note that `Wallet` does not read this persisted data during operation since it always has a copy
+//! in memory
+use crate::KeychainKind;
+use bdk_chain::{keychain::KeychainTracker, ConfirmationTime};
+
+/// `Persist` wraps a [`Backend`] to create a convienient staging area for changes before they are
+/// persisted. Not all changes made to the [`Wallet`] need to be written to disk right away so you
+/// can use [`Persist::stage`] to *stage* it first and then [`Persist::commit`] to finally write it
+/// to disk.
+#[derive(Debug)]
+pub struct Persist<P> {
+    backend: P,
+    stage: ChangeSet,
+}
+
+impl<P> Persist<P> {
+    /// Create a new `Persist` from a [`Backend`]
+    pub fn new(backend: P) -> Self {
+        Self {
+            backend,
+            stage: Default::default(),
+        }
+    }
+
+    /// Stage a `changeset` to later persistence with [`commit`].
+    ///
+    /// [`commit`]: Self::commit
+    pub fn stage(&mut self, changeset: ChangeSet) {
+        self.stage.append(changeset)
+    }
+
+    /// Get the changes that haven't been commited yet
+    pub fn staged(&self) -> &ChangeSet {
+        &self.stage
+    }
+
+    /// Commit the staged changes to the underlying persistence backend.
+    ///
+    /// Retuns a backend defined error if this fails
+    pub fn commit(&mut self) -> Result<(), P::WriteError>
+    where
+        P: Backend,
+    {
+        self.backend.append_changeset(&self.stage)?;
+        self.stage = Default::default();
+        Ok(())
+    }
+}
+
+/// A persistence backend for [`Wallet`]
+///
+/// [`Wallet`]: crate::Wallet
+pub trait Backend {
+    /// The error the backend returns when it fails to write
+    type WriteError: core::fmt::Debug;
+    /// The error the backend returns when it fails to load
+    type LoadError: core::fmt::Debug;
+    /// Appends a new changeset to the persistance backend.
+    ///
+    /// It is up to the backend what it does with this. It could store every changeset in a list or
+    /// it insert the actual changes to a more structured database. All it needs to guarantee is
+    /// that [`load_into_keychain_tracker`] restores a keychain tracker to what it should be if all
+    /// changesets had been applied sequentially.
+    ///
+    /// [`load_into_keychain_tracker`]: Self::load_into_keychain_tracker
+    fn append_changeset(&mut self, changeset: &ChangeSet) -> Result<(), Self::WriteError>;
+
+    /// Applies all the changesets the backend has received to `tracker`.
+    fn load_into_keychain_tracker(
+        &mut self,
+        tracker: &mut KeychainTracker<KeychainKind, ConfirmationTime>,
+    ) -> Result<(), Self::LoadError>;
+}
+
+#[cfg(feature = "file-store")]
+mod file_store {
+    use super::*;
+    use bdk_chain::file_store::{IterError, KeychainStore};
+
+    type FileStore = KeychainStore<KeychainKind, ConfirmationTime>;
+
+    impl Backend for FileStore {
+        type WriteError = std::io::Error;
+        type LoadError = IterError;
+        fn append_changeset(&mut self, changeset: &ChangeSet) -> Result<(), Self::WriteError> {
+            self.append_changeset(changeset)
+        }
+        fn load_into_keychain_tracker(
+            &mut self,
+            tracker: &mut KeychainTracker<KeychainKind, ConfirmationTime>,
+        ) -> Result<(), Self::LoadError> {
+            self.load_into_keychain_tracker(tracker)
+        }
+    }
+}
+
+impl Backend for () {
+    type WriteError = ();
+    type LoadError = ();
+    fn append_changeset(&mut self, _changeset: &ChangeSet) -> Result<(), Self::WriteError> {
+        Ok(())
+    }
+    fn load_into_keychain_tracker(
+        &mut self,
+        _tracker: &mut KeychainTracker<KeychainKind, ConfirmationTime>,
+    ) -> Result<(), Self::LoadError> {
+        Ok(())
+    }
+}
+
+#[cfg(feature = "file-store")]
+pub use file_store::*;
+
+use super::ChangeSet;

src/wallet/tx_builder.rs

@@ -46,6 +46,7 @@ use bitcoin::util::psbt::{self, PartiallySignedTransaction as Psbt};
 use bitcoin::{LockTime, OutPoint, Script, Sequence, Transaction};
 
 use super::coin_selection::{CoinSelectionAlgorithm, DefaultCoinSelectionAlgorithm};
+use super::persist;
 use crate::{
     types::{FeeRate, KeychainKind, LocalUtxo, WeightedUtxo},
     TransactionDetails,
@@ -116,8 +117,8 @@ impl TxBuilderContext for BumpFee {}
 /// [`finish`]: Self::finish
 /// [`coin_selection`]: Self::coin_selection
 #[derive(Debug)]
-pub struct TxBuilder<'a, Cs, Ctx> {
+pub struct TxBuilder<'a, D, Cs, Ctx> {
-    pub(crate) wallet: Rc<RefCell<&'a mut Wallet>>,
+    pub(crate) wallet: Rc<RefCell<&'a mut Wallet<D>>>,
     pub(crate) params: TxParams,
     pub(crate) coin_selection: Cs,
     pub(crate) phantom: PhantomData<Ctx>,
@@ -168,7 +169,7 @@ impl Default for FeePolicy {
     }
 }
 
-impl<'a, Cs: Clone, Ctx> Clone for TxBuilder<'a, Cs, Ctx> {
+impl<'a, D, Cs: Clone, Ctx> Clone for TxBuilder<'a, D, Cs, Ctx> {
     fn clone(&self) -> Self {
         TxBuilder {
             wallet: self.wallet.clone(),
@@ -180,7 +181,7 @@ impl<'a, Cs: Clone, Ctx> Clone for TxBuilder<'a, Cs, Ctx> {
 }
 
 // methods supported by both contexts, for any CoinSelectionAlgorithm
-impl<'a, Cs: CoinSelectionAlgorithm, Ctx: TxBuilderContext> TxBuilder<'a, Cs, Ctx> {
+impl<'a, D, Cs: CoinSelectionAlgorithm, Ctx: TxBuilderContext> TxBuilder<'a, D, Cs, Ctx> {
     /// Set a custom fee rate
     pub fn fee_rate(&mut self, fee_rate: FeeRate) -> &mut Self {
         self.params.fee_policy = Some(FeePolicy::FeeRate(fee_rate));
@@ -508,7 +509,7 @@ impl<'a, Cs: CoinSelectionAlgorithm, Ctx: TxBuilderContext> TxBuilder<'a, Cs, Ct
     pub fn coin_selection<P: CoinSelectionAlgorithm>(
         self,
         coin_selection: P,
-    ) -> TxBuilder<'a, P, Ctx> {
+    ) -> TxBuilder<'a, D, P, Ctx> {
         TxBuilder {
             wallet: self.wallet,
             params: self.params,
@@ -522,7 +523,10 @@ impl<'a, Cs: CoinSelectionAlgorithm, Ctx: TxBuilderContext> TxBuilder<'a, Cs, Ct
     /// Returns the [`BIP174`] "PSBT" and summary details about the transaction.
     ///
     /// [`BIP174`]: https://github.com/bitcoin/bips/blob/master/bip-0174.mediawiki
-    pub fn finish(self) -> Result<(Psbt, TransactionDetails), Error> {
+    pub fn finish(self) -> Result<(Psbt, TransactionDetails), Error>
+    where
+        D: persist::Backend,
+    {
         self.wallet
             .borrow_mut()
             .create_tx(self.coin_selection, self.params)
@@ -573,7 +577,7 @@ impl<'a, Cs: CoinSelectionAlgorithm, Ctx: TxBuilderContext> TxBuilder<'a, Cs, Ct
     }
 }
 
-impl<'a, Cs: CoinSelectionAlgorithm> TxBuilder<'a, Cs, CreateTx> {
+impl<'a, D, Cs: CoinSelectionAlgorithm> TxBuilder<'a, D, Cs, CreateTx> {
     /// Replace the recipients already added with a new list
     pub fn set_recipients(&mut self, recipients: Vec<(Script, u64)>) -> &mut Self {
         self.params.recipients = recipients;
@@ -644,7 +648,7 @@ impl<'a, Cs: CoinSelectionAlgorithm> TxBuilder<'a, Cs, CreateTx> {
 }
 
 // methods supported only by bump_fee
-impl<'a> TxBuilder<'a, DefaultCoinSelectionAlgorithm, BumpFee> {
+impl<'a, D> TxBuilder<'a, D, DefaultCoinSelectionAlgorithm, BumpFee> {
     /// Explicitly tells the wallet that it is allowed to reduce the amount of the output matching this
     /// `script_pubkey` in order to bump the transaction fee. Without specifying this the wallet
     /// will attempt to find a change output to shrink instead.
@@ -699,14 +703,8 @@ impl TxOrdering {
             TxOrdering::Untouched => {}
             TxOrdering::Shuffle => {
                 use rand::seq::SliceRandom;
-                #[cfg(test)]
-                use rand::SeedableRng;
-
-                #[cfg(not(test))]
                 let mut rng = rand::thread_rng();
-                #[cfg(test)]
+                tx.input.shuffle(&mut rng);
-                let mut rng = rand::rngs::StdRng::seed_from_u64(12345);
-
                 tx.output.shuffle(&mut rng);
             }
             TxOrdering::Bip69Lexicographic => {
@@ -818,10 +816,20 @@ mod test {
         let original_tx = ordering_test_tx!();
         let mut tx = original_tx.clone();
 
-        TxOrdering::Shuffle.sort_tx(&mut tx);
+        (0..40)
+            .find(|_| {
+                TxOrdering::Shuffle.sort_tx(&mut tx);
+                original_tx.input != tx.input
+            })
+            .expect("it should have moved the inputs at least once");
 
-        assert_eq!(original_tx.input, tx.input);
+        let mut tx = original_tx.clone();
-        assert_ne!(original_tx.output, tx.output);
+        (0..40)
+            .find(|_| {
+                TxOrdering::Shuffle.sort_tx(&mut tx);
+                original_tx.output != tx.output
+            })
+            .expect("it should have moved the outputs at least once");
     }
 
     #[test]

tests/common.rs

@@ -6,7 +6,7 @@ use bitcoin::{BlockHash, Network, Transaction, TxOut};
 
 /// Return a fake wallet that appears to be funded for testing.
 pub fn get_funded_wallet(descriptor: &str) -> (Wallet, bitcoin::Txid) {
-    let mut wallet = Wallet::new(descriptor, None, Network::Regtest).unwrap();
+    let mut wallet = Wallet::new_no_persist(descriptor, None, Network::Regtest).unwrap();
     let address = wallet.get_address(AddressIndex::New).address;
 
     let tx = Transaction {

tests/wallet.rs

@@ -880,7 +880,7 @@ fn test_create_tx_policy_path_required() {
 #[test]
 fn test_create_tx_policy_path_no_csv() {
     let descriptors = get_test_wpkh();
-    let mut wallet = Wallet::new(descriptors, None, Network::Regtest).unwrap();
+    let mut wallet = Wallet::new_no_persist(descriptors, None, Network::Regtest).unwrap();
 
     let tx = Transaction {
         version: 0,
@@ -1594,7 +1594,7 @@ fn test_bump_fee_absolute_add_input() {
     let (mut wallet, _) = get_funded_wallet(get_test_wpkh());
     receive_output_in_latest_block(&mut wallet, 25_000);
     let addr = Address::from_str("2N1Ffz3WaNzbeLFBb51xyFMHYSEUXcbiSoX").unwrap();
-    let mut builder = wallet.build_tx();
+    let mut builder = wallet.build_tx().coin_selection(LargestFirstCoinSelection);
     builder
         .add_recipient(addr.script_pubkey(), 45_000)
         .enable_rbf();
@@ -1810,7 +1810,7 @@ fn test_bump_fee_absolute_force_add_input() {
     let incoming_op = receive_output_in_latest_block(&mut wallet, 25_000);
 
     let addr = Address::from_str("2N1Ffz3WaNzbeLFBb51xyFMHYSEUXcbiSoX").unwrap();
-    let mut builder = wallet.build_tx();
+    let mut builder = wallet.build_tx().coin_selection(LargestFirstCoinSelection);
     builder
         .add_recipient(addr.script_pubkey(), 45_000)
         .enable_rbf();
@@ -2216,7 +2216,7 @@ fn test_sign_nonstandard_sighash() {
 
 #[test]
 fn test_unused_address() {
-    let mut wallet = Wallet::new("wpkh(tpubEBr4i6yk5nf5DAaJpsi9N2pPYBeJ7fZ5Z9rmN4977iYLCGco1VyjB9tvvuvYtfZzjD5A8igzgw3HeWeeKFmanHYqksqZXYXGsw5zjnj7KM9/*)",
+    let mut wallet = Wallet::new_no_persist("wpkh(tpubEBr4i6yk5nf5DAaJpsi9N2pPYBeJ7fZ5Z9rmN4977iYLCGco1VyjB9tvvuvYtfZzjD5A8igzgw3HeWeeKFmanHYqksqZXYXGsw5zjnj7KM9/*)",
                                  None, Network::Testnet).unwrap();
 
     assert_eq!(
@@ -2232,7 +2232,7 @@ fn test_unused_address() {
 #[test]
 fn test_next_unused_address() {
     let descriptor = "wpkh(tpubEBr4i6yk5nf5DAaJpsi9N2pPYBeJ7fZ5Z9rmN4977iYLCGco1VyjB9tvvuvYtfZzjD5A8igzgw3HeWeeKFmanHYqksqZXYXGsw5zjnj7KM9/*)";
-    let mut wallet = Wallet::new(descriptor, None, Network::Testnet).unwrap();
+    let mut wallet = Wallet::new_no_persist(descriptor, None, Network::Testnet).unwrap();
     assert_eq!(wallet.derivation_index(KeychainKind::External), None);
 
     assert_eq!(
@@ -2258,7 +2258,7 @@ fn test_next_unused_address() {
 
 #[test]
 fn test_peek_address_at_index() {
-    let mut wallet = Wallet::new("wpkh(tpubEBr4i6yk5nf5DAaJpsi9N2pPYBeJ7fZ5Z9rmN4977iYLCGco1VyjB9tvvuvYtfZzjD5A8igzgw3HeWeeKFmanHYqksqZXYXGsw5zjnj7KM9/*)",
+    let mut wallet = Wallet::new_no_persist("wpkh(tpubEBr4i6yk5nf5DAaJpsi9N2pPYBeJ7fZ5Z9rmN4977iYLCGco1VyjB9tvvuvYtfZzjD5A8igzgw3HeWeeKFmanHYqksqZXYXGsw5zjnj7KM9/*)",
                                  None, Network::Testnet).unwrap();
 
     assert_eq!(
@@ -2290,7 +2290,7 @@ fn test_peek_address_at_index() {
 
 #[test]
 fn test_peek_address_at_index_not_derivable() {
-    let mut wallet = Wallet::new("wpkh(tpubEBr4i6yk5nf5DAaJpsi9N2pPYBeJ7fZ5Z9rmN4977iYLCGco1VyjB9tvvuvYtfZzjD5A8igzgw3HeWeeKFmanHYqksqZXYXGsw5zjnj7KM9/1)",
+    let mut wallet = Wallet::new_no_persist("wpkh(tpubEBr4i6yk5nf5DAaJpsi9N2pPYBeJ7fZ5Z9rmN4977iYLCGco1VyjB9tvvuvYtfZzjD5A8igzgw3HeWeeKFmanHYqksqZXYXGsw5zjnj7KM9/1)",
                                  None, Network::Testnet).unwrap();
 
     assert_eq!(
@@ -2311,7 +2311,7 @@ fn test_peek_address_at_index_not_derivable() {
 
 #[test]
 fn test_returns_index_and_address() {
-    let mut wallet = Wallet::new("wpkh(tpubEBr4i6yk5nf5DAaJpsi9N2pPYBeJ7fZ5Z9rmN4977iYLCGco1VyjB9tvvuvYtfZzjD5A8igzgw3HeWeeKFmanHYqksqZXYXGsw5zjnj7KM9/*)",
+    let mut wallet = Wallet::new_no_persist("wpkh(tpubEBr4i6yk5nf5DAaJpsi9N2pPYBeJ7fZ5Z9rmN4977iYLCGco1VyjB9tvvuvYtfZzjD5A8igzgw3HeWeeKFmanHYqksqZXYXGsw5zjnj7KM9/*)",
                                  None, Network::Testnet).unwrap();
 
     // new index 0
@@ -2369,7 +2369,7 @@ fn test_sending_to_bip350_bech32m_address() {
 fn test_get_address() {
     use bdk::descriptor::template::Bip84;
     let key = bitcoin::util::bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
-    let mut wallet = Wallet::new(
+    let mut wallet = Wallet::new_no_persist(
         Bip84(key, KeychainKind::External),
         Some(Bip84(key, KeychainKind::Internal)),
         Network::Regtest,
@@ -2395,14 +2395,14 @@ fn test_get_address() {
     );
 
     let mut wallet =
-        Wallet::new(Bip84(key, KeychainKind::External), None, Network::Regtest).unwrap();
+        Wallet::new_no_persist(Bip84(key, KeychainKind::External), None, Network::Regtest).unwrap();
 
     assert_eq!(
         wallet.get_internal_address(AddressIndex::New),
         AddressInfo {
             index: 0,
             address: Address::from_str("bcrt1qrhgaqu0zvf5q2d0gwwz04w0dh0cuehhqvzpp4w").unwrap(),
-            keychain: KeychainKind::Internal,
+            keychain: KeychainKind::External,
         },
         "when there's no internal descriptor it should just use external"
     );
@@ -2415,7 +2415,7 @@ fn test_get_address_no_reuse_single_descriptor() {
 
     let key = bitcoin::util::bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
     let mut wallet =
-        Wallet::new(Bip84(key, KeychainKind::External), None, Network::Regtest).unwrap();
+        Wallet::new_no_persist(Bip84(key, KeychainKind::External), None, Network::Regtest).unwrap();
 
     let mut used_set = HashSet::new();
 
@@ -2876,7 +2876,8 @@ fn test_taproot_sign_derive_index_from_psbt() {
     let (mut psbt, _) = builder.finish().unwrap();
 
     // re-create the wallet with an empty db
-    let wallet_empty = Wallet::new(get_test_tr_single_sig_xprv(), None, Network::Regtest).unwrap();
+    let wallet_empty =
+        Wallet::new_no_persist(get_test_tr_single_sig_xprv(), None, Network::Regtest).unwrap();
 
     // signing with an empty db means that we will only look at the psbt to infer the
     // derivation index
@@ -2976,7 +2977,7 @@ fn test_taproot_sign_non_default_sighash() {
 #[test]
 fn test_spend_coinbase() {
     let descriptor = get_test_wpkh();
-    let mut wallet = Wallet::new(descriptor, None, Network::Regtest).unwrap();
+    let mut wallet = Wallet::new_no_persist(descriptor, None, Network::Regtest).unwrap();
 
     let confirmation_height = 5;
     wallet