frost/bdk
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(electrum)!: change signature of ElectrumExt

Browse Source 
We remove `ElectrumUpdate` and return tuples instead for `ElectrumExt`
methods. We introduce the `IncompleteTxGraph` structure to specifically
hodl the incomplete `TxGraph`.

This change is motivated by @LLFourn's comment: 794bf37e63 (r1305432603)
This commit is contained in:
志宇 2023-08-26 20:29:46 +08:00
parent a28748c339
commit 32c40ac939
No known key found for this signature in database
GPG Key ID: F6345C9837C2BDE8
4 changed files with 93 additions and 103 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

144
crates/electrum/src/electrum_ext.rs
View File

@ -14,86 +14,63 @@ use std::{
/// We assume that a block of this depth and deeper cannot be reorged. /// We assume that a block of this depth and deeper cannot be reorged.
const ASSUME_FINAL_DEPTH: u32 = 8; const ASSUME_FINAL_DEPTH: u32 = 8;
/// Represents an update fetched from an Electrum server, but excludes full /// Represents a [`TxGraph`] update fetched from an Electrum server, but excludes full transactions.
/// transactions.
/// ///
/// To provide a complete update to [`TxGraph`], you'll need to call [`Self::missing_full_txs`] to /// To provide a complete update to [`TxGraph`], you'll need to call [`Self::missing_full_txs`] to
/// determine the full transactions missing from [`TxGraph`]. Then call [`Self::finalize`] to fetch /// determine the full transactions missing from [`TxGraph`]. Then call [`Self::finalize`] to
/// the full transactions from Electrum and finalize the update. /// fetch the full transactions from Electrum and finalize the update.
#[derive(Debug, Clone)] #[derive(Debug, Default, Clone)]
pub struct ElectrumUpdate<K, A> { pub struct IncompleteTxGraph<A>(HashMap<Txid, BTreeSet<A>>);
/// Map of [`Txid`]s to associated [`Anchor`]s.
pub graph_update: HashMap<Txid, BTreeSet<A>>,
/// The latest chain tip, as seen by the Electrum server.
pub new_tip: local_chain::CheckPoint,
/// Last-used index update for [`KeychainTxOutIndex`](bdk_chain::keychain::KeychainTxOutIndex).
pub keychain_update: BTreeMap<K, u32>,
}
impl<K, A: Anchor> ElectrumUpdate<K, A> {
fn new(new_tip: local_chain::CheckPoint) -> Self {
Self {
new_tip,
graph_update: HashMap::new(),
keychain_update: BTreeMap::new(),
}
}
impl<A: Anchor> IncompleteTxGraph<A> {
/// Determine the full transactions that are missing from `graph`. /// Determine the full transactions that are missing from `graph`.
/// ///
/// Refer to [`ElectrumUpdate`]. /// Refer to [`IncompleteTxGraph`] for more.
pub fn missing_full_txs<A2>(&self, graph: &TxGraph<A2>) -> Vec<Txid> { pub fn missing_full_txs<A2>(&self, graph: &TxGraph<A2>) -> Vec<Txid> {
self.graph_update self.0
.keys() .keys()
.filter(move |&&txid| graph.as_ref().get_tx(txid).is_none()) .filter(move |&&txid| graph.as_ref().get_tx(txid).is_none())
.cloned() .cloned()
.collect() .collect()
} }
/// Finalizes update with `missing` txids to fetch from `client`. /// Finalizes the [`TxGraph`] update by fetching `missing` txids from the `client`.
/// ///
/// Refer to [`ElectrumUpdate`]. /// Refer to [`IncompleteTxGraph`] for more.
pub fn finalize( pub fn finalize(
self, self,
client: &Client, client: &Client,
seen_at: Option<u64>, seen_at: Option<u64>,
missing: Vec<Txid>, missing: Vec<Txid>,
) -> Result<(TxGraph<A>, BTreeMap<K, u32>, local_chain::CheckPoint), Error> { ) -> Result<TxGraph<A>, Error> {
let new_txs = client.batch_transaction_get(&missing)?; let new_txs = client.batch_transaction_get(&missing)?;
let mut graph_update = TxGraph::<A>::new(new_txs); let mut graph = TxGraph::<A>::new(new_txs);
for (txid, anchors) in self.graph_update { for (txid, anchors) in self.0 {
if let Some(seen_at) = seen_at { if let Some(seen_at) = seen_at {
let _ = graph_update.insert_seen_at(txid, seen_at); let _ = graph.insert_seen_at(txid, seen_at);
} }
for anchor in anchors { for anchor in anchors {
let _ = graph_update.insert_anchor(txid, anchor); let _ = graph.insert_anchor(txid, anchor);
} }
} }
Ok((graph_update, self.keychain_update, self.new_tip)) Ok(graph)
} }
} }
impl<K> ElectrumUpdate<K, ConfirmationHeightAnchor> { impl IncompleteTxGraph<ConfirmationHeightAnchor> {
/// Finalizes the [`ElectrumUpdate`] with `new_txs` and anchors of type /// Finalizes the [`IncompleteTxGraph`] with `new_txs` and anchors of type
/// [`ConfirmationTimeAnchor`]. /// [`ConfirmationTimeAnchor`].
/// ///
/// **Note:** The confirmation time might not be precisely correct if there has been a reorg. /// **Note:** The confirmation time might not be precisely correct if there has been a reorg.
/// Electrum's API intends that we use the merkle proof API, we should change `bdk_electrum` to /// Electrum's API intends that we use the merkle proof API, we should change `bdk_electrum` to
/// use it. /// use it.
pub fn finalize_as_confirmation_time( pub fn finalize_with_confirmation_time(
self, self,
client: &Client, client: &Client,
seen_at: Option<u64>, seen_at: Option<u64>,
missing: Vec<Txid>, missing: Vec<Txid>,
) -> Result< ) -> Result<TxGraph<ConfirmationTimeAnchor>, Error> {
( let graph = self.finalize(client, seen_at, missing)?;
TxGraph<ConfirmationTimeAnchor>,
BTreeMap<K, u32>,
local_chain::CheckPoint,
),
Error,
> {
let (graph, keychain_update, update_tip) = self.finalize(client, seen_at, missing)?;
let relevant_heights = { let relevant_heights = {
let mut visited_heights = HashSet::new(); let mut visited_heights = HashSet::new();
@ -117,7 +94,7 @@ impl<K> ElectrumUpdate<K, ConfirmationHeightAnchor> {
.collect::<HashMap<u32, u64>>(); .collect::<HashMap<u32, u64>>();
let graph_changeset = { let graph_changeset = {
let old_changeset = TxGraph::default().apply_update(graph.clone()); let old_changeset = TxGraph::default().apply_update(graph);
tx_graph::ChangeSet { tx_graph::ChangeSet {
txs: old_changeset.txs, txs: old_changeset.txs,
txouts: old_changeset.txouts, txouts: old_changeset.txouts,
@ -139,16 +116,16 @@ impl<K> ElectrumUpdate<K, ConfirmationHeightAnchor> {
} }
}; };
let mut update = TxGraph::default(); let mut new_graph = TxGraph::default();
update.apply_changeset(graph_changeset); new_graph.apply_changeset(graph_changeset);
Ok(new_graph)
Ok((update, keychain_update, update_tip))
} }
} }
/// Trait to extend [`Client`] functionality. /// Trait to extend [`Client`] functionality.
pub trait ElectrumExt<A> { pub trait ElectrumExt<A> {
/// Scan the blockchain (via electrum) for the data specified and returns a [`ElectrumUpdate`]. /// Scan the blockchain (via electrum) for the data specified and returns updates for
/// [`bdk_chain`] data structures.
/// ///
/// - `prev_tip`: the most recent blockchain tip present locally /// - `prev_tip`: the most recent blockchain tip present locally
/// - `keychain_spks`: keychains that we want to scan transactions for /// - `keychain_spks`: keychains that we want to scan transactions for
@ -159,6 +136,7 @@ pub trait ElectrumExt<A> {
/// The scan for each keychain stops after a gap of `stop_gap` script pubkeys with no associated /// The scan for each keychain stops after a gap of `stop_gap` script pubkeys with no associated
/// transactions. `batch_size` specifies the max number of script pubkeys to request for in a /// transactions. `batch_size` specifies the max number of script pubkeys to request for in a
/// single batch request. /// single batch request.
#[allow(clippy::type_complexity)]
fn scan<K: Ord + Clone>( fn scan<K: Ord + Clone>(
&self, &self,
prev_tip: Option<CheckPoint>, prev_tip: Option<CheckPoint>,
@ -167,7 +145,7 @@ pub trait ElectrumExt<A> {
outpoints: impl IntoIterator<Item = OutPoint>, outpoints: impl IntoIterator<Item = OutPoint>,
stop_gap: usize, stop_gap: usize,
batch_size: usize, batch_size: usize,
) -> Result<ElectrumUpdate<K, A>, Error>; ) -> Result<(local_chain::Update, IncompleteTxGraph<A>, BTreeMap<K, u32>), Error>;
/// Convenience method to call [`scan`] without requiring a keychain. /// Convenience method to call [`scan`] without requiring a keychain.
/// ///
@ -179,20 +157,22 @@ pub trait ElectrumExt<A> {
txids: impl IntoIterator<Item = Txid>, txids: impl IntoIterator<Item = Txid>,
outpoints: impl IntoIterator<Item = OutPoint>, outpoints: impl IntoIterator<Item = OutPoint>,
batch_size: usize, batch_size: usize,
) -> Result<ElectrumUpdate<(), A>, Error> { ) -> Result<(local_chain::Update, IncompleteTxGraph<A>), Error> {
let spk_iter = misc_spks let spk_iter = misc_spks
.into_iter() .into_iter()
.enumerate() .enumerate()
.map(|(i, spk)| (i as u32, spk)); .map(|(i, spk)| (i as u32, spk));
self.scan( let (chain, graph, _) = self.scan(
prev_tip, prev_tip,
[((), spk_iter)].into(), [((), spk_iter)].into(),
txids, txids,
outpoints, outpoints,
usize::MAX, usize::MAX,
batch_size, batch_size,
) )?;
Ok((chain, graph))
} }
} }
@ -205,7 +185,14 @@ impl ElectrumExt<ConfirmationHeightAnchor> for Client {
outpoints: impl IntoIterator<Item = OutPoint>, outpoints: impl IntoIterator<Item = OutPoint>,
stop_gap: usize, stop_gap: usize,
batch_size: usize, batch_size: usize,
) -> Result<ElectrumUpdate<K, ConfirmationHeightAnchor>, Error> { ) -> Result<
(
local_chain::Update,
IncompleteTxGraph<ConfirmationHeightAnchor>,
BTreeMap<K, u32>,
),
Error,
> {
let mut request_spks = keychain_spks let mut request_spks = keychain_spks
.into_iter() .into_iter()
.map(|(k, s)| (k, s.into_iter())) .map(|(k, s)| (k, s.into_iter()))
@ -217,9 +204,8 @@ impl ElectrumExt<ConfirmationHeightAnchor> for Client {
let update = loop { let update = loop {
let (tip, _) = construct_update_tip(self, prev_tip.clone())?; let (tip, _) = construct_update_tip(self, prev_tip.clone())?;
let mut update = ElectrumUpdate::<K, ConfirmationHeightAnchor>::new(tip.clone()); let mut graph_update = IncompleteTxGraph::<ConfirmationHeightAnchor>::default();
let cps = update let cps = tip
.new_tip
.iter() .iter()
.take(10) .take(10)
.map(|cp| (cp.height(), cp)) .map(|cp| (cp.height(), cp))
@ -230,7 +216,7 @@ impl ElectrumExt<ConfirmationHeightAnchor> for Client {
scanned_spks.append(&mut populate_with_spks( scanned_spks.append(&mut populate_with_spks(
self, self,
&cps, &cps,
&mut update, &mut graph_update,
&mut scanned_spks &mut scanned_spks
.iter() .iter()
.map(|(i, (spk, _))| (i.clone(), spk.clone())), .map(|(i, (spk, _))| (i.clone(), spk.clone())),
@ -243,7 +229,7 @@ impl ElectrumExt<ConfirmationHeightAnchor> for Client {
populate_with_spks( populate_with_spks(
self, self,
&cps, &cps,
&mut update, &mut graph_update,
keychain_spks, keychain_spks,
stop_gap, stop_gap,
batch_size, batch_size,
@ -254,10 +240,14 @@ impl ElectrumExt<ConfirmationHeightAnchor> for Client {
} }
} }
populate_with_txids(self, &cps, &mut update, &mut txids.iter().cloned())?; populate_with_txids(self, &cps, &mut graph_update, &mut txids.iter().cloned())?;
let _txs = let _txs = populate_with_outpoints(
populate_with_outpoints(self, &cps, &mut update, &mut outpoints.iter().cloned())?; self,
&cps,
&mut graph_update,
&mut outpoints.iter().cloned(),
)?;
// check for reorgs during scan process // check for reorgs during scan process
let server_blockhash = self.block_header(tip.height() as usize)?.block_hash(); let server_blockhash = self.block_header(tip.height() as usize)?.block_hash();
@ -265,7 +255,12 @@ impl ElectrumExt<ConfirmationHeightAnchor> for Client {
continue; // reorg continue; // reorg
} }
update.keychain_update = request_spks let chain_update = local_chain::Update {
tip,
introduce_older_blocks: true,
};
let keychain_update = request_spks
.into_keys() .into_keys()
.filter_map(|k| { .filter_map(|k| {
scanned_spks scanned_spks
@ -275,7 +270,8 @@ impl ElectrumExt<ConfirmationHeightAnchor> for Client {
.map(|((_, i), _)| (k, *i)) .map(|((_, i), _)| (k, *i))
}) })
.collect::<BTreeMap<_, _>>(); .collect::<BTreeMap<_, _>>();
break update;
break (chain_update, graph_update, keychain_update);
}; };
Ok(update) Ok(update)
@ -399,10 +395,10 @@ fn determine_tx_anchor(
} }
} }
fn populate_with_outpoints<K>( fn populate_with_outpoints(
client: &Client, client: &Client,
cps: &BTreeMap<u32, CheckPoint>, cps: &BTreeMap<u32, CheckPoint>,
update: &mut ElectrumUpdate<K, ConfirmationHeightAnchor>, graph_update: &mut IncompleteTxGraph<ConfirmationHeightAnchor>,
outpoints: &mut impl Iterator<Item = OutPoint>, outpoints: &mut impl Iterator<Item = OutPoint>,
) -> Result<HashMap<Txid, Transaction>, Error> { ) -> Result<HashMap<Txid, Transaction>, Error> {
let mut full_txs = HashMap::new(); let mut full_txs = HashMap::new();
@ -451,7 +447,7 @@ fn populate_with_outpoints<K>(
}; };
let anchor = determine_tx_anchor(cps, res.height, res.tx_hash); let anchor = determine_tx_anchor(cps, res.height, res.tx_hash);
let tx_entry = update.graph_update.entry(res.tx_hash).or_default(); let tx_entry = graph_update.0.entry(res.tx_hash).or_default();
if let Some(anchor) = anchor { if let Some(anchor) = anchor {
tx_entry.insert(anchor); tx_entry.insert(anchor);
} }
@ -460,10 +456,10 @@ fn populate_with_outpoints<K>(
Ok(full_txs) Ok(full_txs)
} }
fn populate_with_txids<K>( fn populate_with_txids(
client: &Client, client: &Client,
cps: &BTreeMap<u32, CheckPoint>, cps: &BTreeMap<u32, CheckPoint>,
update: &mut ElectrumUpdate<K, ConfirmationHeightAnchor>, graph_update: &mut IncompleteTxGraph<ConfirmationHeightAnchor>,
txids: &mut impl Iterator<Item = Txid>, txids: &mut impl Iterator<Item = Txid>,
) -> Result<(), Error> { ) -> Result<(), Error> {
for txid in txids { for txid in txids {
@ -488,7 +484,7 @@ fn populate_with_txids<K>(
None => continue, None => continue,
}; };
let tx_entry = update.graph_update.entry(txid).or_default(); let tx_entry = graph_update.0.entry(txid).or_default();
if let Some(anchor) = anchor { if let Some(anchor) = anchor {
tx_entry.insert(anchor); tx_entry.insert(anchor);
} }
@ -496,10 +492,10 @@ fn populate_with_txids<K>(
Ok(()) Ok(())
} }
fn populate_with_spks<K, I: Ord + Clone>( fn populate_with_spks<I: Ord + Clone>(
client: &Client, client: &Client,
cps: &BTreeMap<u32, CheckPoint>, cps: &BTreeMap<u32, CheckPoint>,
update: &mut ElectrumUpdate<K, ConfirmationHeightAnchor>, graph_update: &mut IncompleteTxGraph<ConfirmationHeightAnchor>,
spks: &mut impl Iterator<Item = (I, ScriptBuf)>, spks: &mut impl Iterator<Item = (I, ScriptBuf)>,
stop_gap: usize, stop_gap: usize,
batch_size: usize, batch_size: usize,
@ -532,7 +528,7 @@ fn populate_with_spks<K, I: Ord + Clone>(
} }
for tx in spk_history { for tx in spk_history {
let tx_entry = update.graph_update.entry(tx.tx_hash).or_default(); let tx_entry = graph_update.0.entry(tx.tx_hash).or_default();
if let Some(anchor) = determine_tx_anchor(cps, tx.height, tx.tx_hash) { if let Some(anchor) = determine_tx_anchor(cps, tx.height, tx.tx_hash) {
tx_entry.insert(anchor); tx_entry.insert(anchor);
} }

12
crates/electrum/src/lib.rs
View File

@ -1,14 +1,16 @@
//! This crate is used for updating structures of the [`bdk_chain`] crate with data from electrum. //! This crate is used for updating structures of the [`bdk_chain`] crate with data from electrum.
//! //!
//! The star of the show is the [`ElectrumExt::scan`] method, which scans for relevant blockchain //! The star of the show is the [`ElectrumExt::scan`] method, which scans for relevant blockchain
//! data (via electrum) and outputs an [`ElectrumUpdate`]. //! data (via electrum) and outputs updates for [`bdk_chain`] structures as a tuple of form:
//! //!
//! An [`ElectrumUpdate`] only includes `txid`s and no full transactions. The caller is responsible //! ([`bdk_chain::local_chain::Update`], [`IncompleteTxGraph`], `keychain_update`)
//! for obtaining full transactions before applying. This can be done with //!
//! An [`IncompleteTxGraph`] only includes `txid`s and no full transactions. The caller is
//! responsible for obtaining full transactions before applying. This can be done with
//! these steps: //! these steps:
//! //!
//! 1. Determine which full transactions are missing. The method [`missing_full_txs`] of //! 1. Determine which full transactions are missing. The method [`missing_full_txs`] of
//! [`ElectrumUpdate`] can be used. //! [`IncompleteTxGraph`] can be used.
//! //!
//! 2. Obtaining the full transactions. To do this via electrum, the method //! 2. Obtaining the full transactions. To do this via electrum, the method
//! [`batch_transaction_get`] can be used. //! [`batch_transaction_get`] can be used.
@ -16,7 +18,7 @@
//! Refer to [`bdk_electrum_example`] for a complete example. //! Refer to [`bdk_electrum_example`] for a complete example.
//! //!
//! [`ElectrumClient::scan`]: electrum_client::ElectrumClient::scan //! [`ElectrumClient::scan`]: electrum_client::ElectrumClient::scan
//! [`missing_full_txs`]: ElectrumUpdate::missing_full_txs //! [`missing_full_txs`]: IncompleteTxGraph::missing_full_txs
//! [`batch_transaction_get`]: electrum_client::ElectrumApi::batch_transaction_get //! [`batch_transaction_get`]: electrum_client::ElectrumApi::batch_transaction_get
//! [`bdk_electrum_example`]: https://github.com/LLFourn/bdk_core_staging/tree/master/bdk_electrum_example //! [`bdk_electrum_example`]: https://github.com/LLFourn/bdk_core_staging/tree/master/bdk_electrum_example

22
example-crates/example_electrum/src/main.rs
View File

@ -13,7 +13,7 @@ use bdk_chain::{
}; };
use bdk_electrum::{ use bdk_electrum::{
electrum_client::{self, ElectrumApi}, electrum_client::{self, ElectrumApi},
ElectrumExt, ElectrumUpdate, ElectrumExt,
}; };
use example_cli::{ use example_cli::{
anyhow::{self, Context}, anyhow::{self, Context},
@ -251,20 +251,18 @@ fn main() -> anyhow::Result<()> {
// drop lock on graph and chain // drop lock on graph and chain
drop((graph, chain)); drop((graph, chain));
let update = client let (chain_update, graph_update) = client
.scan_without_keychain(tip, spks, txids, outpoints, scan_options.batch_size) .scan_without_keychain(tip, spks, txids, outpoints, scan_options.batch_size)
.context("scanning the blockchain")?; .context("scanning the blockchain")?;
ElectrumUpdate { (chain_update, graph_update, BTreeMap::new())
graph_update: update.graph_update,
new_tip: update.new_tip,
keychain_update: BTreeMap::new(),
}
} }
}; };
let (chain_update, incomplete_graph_update, keychain_update) = response;
let missing_txids = { let missing_txids = {
let graph = &*graph.lock().unwrap(); let graph = &*graph.lock().unwrap();
response.missing_full_txs(graph.graph()) incomplete_graph_update.missing_full_txs(graph.graph())
}; };
let now = std::time::UNIX_EPOCH let now = std::time::UNIX_EPOCH
@ -272,17 +270,13 @@ fn main() -> anyhow::Result<()> {
.expect("must get time") .expect("must get time")
.as_secs(); .as_secs();
let (graph_update, keychain_update, update_tip) = let graph_update = incomplete_graph_update.finalize(&client, Some(now), missing_txids)?;
response.finalize(&client, Some(now), missing_txids)?;
let db_changeset = { let db_changeset = {
let mut chain = chain.lock().unwrap(); let mut chain = chain.lock().unwrap();
let mut graph = graph.lock().unwrap(); let mut graph = graph.lock().unwrap();
let chain = chain.apply_update(local_chain::Update { let chain = chain.apply_update(chain_update)?;
tip: update_tip,
introduce_older_blocks: true,
})?;
let indexed_tx_graph = { let indexed_tx_graph = {
let mut changeset = let mut changeset =

18
example-crates/wallet_electrum/src/main.rs
View File

@ -7,9 +7,9 @@ use std::io::Write;
use std::str::FromStr; use std::str::FromStr;
use bdk::bitcoin::Address; use bdk::bitcoin::Address;
use bdk::wallet::WalletUpdate;
use bdk::SignOptions; use bdk::SignOptions;
use bdk::{bitcoin::Network, wallet::WalletUpdate, Wallet}; use bdk::{bitcoin::Network, Wallet};
use bdk_electrum::bdk_chain::local_chain;
use bdk_electrum::electrum_client::{self, ElectrumApi}; use bdk_electrum::electrum_client::{self, ElectrumApi};
use bdk_electrum::ElectrumExt; use bdk_electrum::ElectrumExt;
use bdk_file_store::Store; use bdk_file_store::Store;
@ -53,21 +53,19 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
}) })
.collect(); .collect();
let electrum_update = client.scan(prev_tip, keychain_spks, None, None, STOP_GAP, BATCH_SIZE)?; let (chain_update, incomplete_graph_update, keychain_update) =
client.scan(prev_tip, keychain_spks, None, None, STOP_GAP, BATCH_SIZE)?;
println!(); println!();
let missing = electrum_update.missing_full_txs(wallet.as_ref()); let missing = incomplete_graph_update.missing_full_txs(wallet.as_ref());
let (graph_update, keychain_update, update_tip) = let graph_update =
electrum_update.finalize_as_confirmation_time(&client, None, missing)?; incomplete_graph_update.finalize_with_confirmation_time(&client, None, missing)?;
let wallet_update = WalletUpdate { let wallet_update = WalletUpdate {
last_active_indices: keychain_update, last_active_indices: keychain_update,
graph: graph_update, graph: graph_update,
chain: Some(local_chain::Update { chain: Some(chain_update),
tip: update_tip,
introduce_older_blocks: true,
}),
}; };
wallet.apply_update(wallet_update)?; wallet.apply_update(wallet_update)?;
wallet.commit()?; wallet.commit()?;