frost/bdk
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge bitcoindevkit/bdk#1413: Introduce universal sync/full-scan structures for spk-based syncing

Browse Source 
c0374a0eeb feat(chain): `SyncRequest` now uses `ExactSizeIterator`s (志宇)
0f94f24aaf feat(esplora)!: update to use new sync/full-scan structures (志宇)
4c52f3e08e feat(wallet): make wallet compatible with sync/full-scan structures (志宇)
cdfec5f907 feat(chain): add sync/full-scan structures for spk-based syncing (志宇)

Pull request description:

  Fixes #1153
  Replaces #1194

  ### Description

  Introduce universal structures that represent sync/full-scan requests/results.

  ### Notes to the reviewers

  This is based on #1194 but is different in the following ways:
  * The functionality to print scan/sync progress is not reduced.
  * `SyncRequest` and `FullScanRequest` is simplified and fields are exposed for more flexibility.

  ### Changelog notice

  * Add universal structures for initiating/receiving sync/full-scan requests/results for spk-based syncing.
  * Updated `bdk_esplora` chain-source to make use of new universal sync/full-scan structures.

  ### Checklists

  #### All Submissions:

  * [x] I've signed all my commits
  * [x] I followed the [contribution guidelines](https://github.com/bitcoindevkit/bdk/blob/master/CONTRIBUTING.md)
  * [x] I ran `cargo fmt` and `cargo clippy` before committing

  #### New Features:

  * [x] I've added tests for the new feature
  * [x] I've added docs for the new feature

ACKs for top commit:
  notmandatory:
    tACK c0374a0eeb

Tree-SHA512: c2ad66d972a6785079bca615dfd128edcedf6b7a02670651a0ab1ce5b5174dd96f54644680eedbf55e3f1955fe5c34f632eadbd3f71d7ffde658753c6c6d42be
This commit is contained in:
志宇
2024-05-01 14:58:41 +08:00
parent ed3ccc1a9d c0374a0eeb
commit 08fac47c29
11 changed files with 704 additions and 264 deletions
Download Patch File Download Diff File Expand all files Collapse all files

137
example-crates/example_esplora/src/main.rs
View File

@@ -1,14 +1,15 @@
use std::{
collections::BTreeMap,
collections::BTreeSet,
io::{self, Write},
sync::Mutex,
};
use bdk_chain::{
bitcoin::{constants::genesis_block, Address, Network, OutPoint, ScriptBuf, Txid},
bitcoin::{constants::genesis_block, Address, Network, Txid},
indexed_tx_graph::{self, IndexedTxGraph},
keychain,
local_chain::{self, LocalChain},
spk_client::{FullScanRequest, SyncRequest},
Append, ConfirmationTimeHeightAnchor,
};
@@ -167,45 +168,34 @@ fn main() -> anyhow::Result<()> {
scan_options,
..
} => {
let local_tip = chain.lock().expect("mutex must not be poisoned").tip();
let keychain_spks = graph
.lock()
.expect("mutex must not be poisoned")
.index
.all_unbounded_spk_iters()
.into_iter()
// This `map` is purely for logging.
.map(|(keychain, iter)| {
let mut first = true;
let spk_iter = iter.inspect(move |(i, _)| {
if first {
eprint!("\nscanning {}: ", keychain);
first = false;
let request = {
let chain_tip = chain.lock().expect("mutex must not be poisoned").tip();
let indexed_graph = &*graph.lock().expect("mutex must not be poisoned");
FullScanRequest::from_keychain_txout_index(chain_tip, &indexed_graph.index)
.inspect_spks_for_all_keychains({
let mut once = BTreeSet::<Keychain>::new();
move |keychain, spk_i, _| {
if once.insert(keychain) {
eprint!("\nscanning {}: ", keychain);
}
eprint!("{} ", spk_i);
// Flush early to ensure we print at every iteration.
let _ = io::stderr().flush();
}
eprint!("{} ", i);
// Flush early to ensure we print at every iteration.
let _ = io::stderr().flush();
});
(keychain, spk_iter)
})
.collect::<BTreeMap<_, _>>();
})
};
// The client scans keychain spks for transaction histories, stopping after `stop_gap`
// is reached. It returns a `TxGraph` update (`graph_update`) and a structure that
// represents the last active spk derivation indices of keychains
// (`keychain_indices_update`).
let mut update = client
.full_scan(
local_tip,
keychain_spks,
*stop_gap,
scan_options.parallel_requests,
)
.full_scan(request, *stop_gap, scan_options.parallel_requests)
.context("scanning for transactions")?;
// We want to keep track of the latest time a transaction was seen unconfirmed.
let now = std::time::UNIX_EPOCH.elapsed().unwrap().as_secs();
let _ = update.tx_graph.update_last_seen_unconfirmed(now);
let _ = update.graph_update.update_last_seen_unconfirmed(now);
let mut graph = graph.lock().expect("mutex must not be poisoned");
let mut chain = chain.lock().expect("mutex must not be poisoned");
@@ -213,11 +203,11 @@ fn main() -> anyhow::Result<()> {
// deriviation indices. Usually before a scan you are on a fresh wallet with no
// addresses derived so we need to derive up to last active addresses the scan found
// before adding the transactions.
(chain.apply_update(update.local_chain)?, {
(chain.apply_update(update.chain_update)?, {
let (_, index_changeset) = graph
.index
.reveal_to_target_multi(&update.last_active_indices);
let mut indexed_tx_graph_changeset = graph.apply_update(update.tx_graph);
let mut indexed_tx_graph_changeset = graph.apply_update(update.graph_update);
indexed_tx_graph_changeset.append(index_changeset.into());
indexed_tx_graph_changeset
})
@@ -241,12 +231,9 @@ fn main() -> anyhow::Result<()> {
unused_spks = false;
}
// Spks, outpoints and txids we want updates on will be accumulated here.
let mut spks: Box<dyn Iterator<Item = ScriptBuf>> = Box::new(core::iter::empty());
let mut outpoints: Box<dyn Iterator<Item = OutPoint>> = Box::new(core::iter::empty());
let mut txids: Box<dyn Iterator<Item = Txid>> = Box::new(core::iter::empty());
let local_tip = chain.lock().expect("mutex must not be poisoned").tip();
// Spks, outpoints and txids we want updates on will be accumulated here.
let mut request = SyncRequest::from_chain_tip(local_tip.clone());
// Get a short lock on the structures to get spks, utxos, and txs that we are interested
// in.
@@ -260,12 +247,12 @@ fn main() -> anyhow::Result<()> {
.revealed_spks(..)
.map(|(k, i, spk)| (k.to_owned(), i, spk.to_owned()))
.collect::<Vec<_>>();
spks = Box::new(spks.chain(all_spks.into_iter().map(|(k, i, spk)| {
eprintln!("scanning {}:{}", k, i);
request = request.chain_spks(all_spks.into_iter().map(|(k, i, spk)| {
eprint!("scanning {}:{}", k, i);
// Flush early to ensure we print at every iteration.
let _ = io::stderr().flush();
spk
})));
}));
}
if unused_spks {
let unused_spks = graph
@@ -273,17 +260,18 @@ fn main() -> anyhow::Result<()> {
.unused_spks()
.map(|(k, i, spk)| (k, i, spk.to_owned()))
.collect::<Vec<_>>();
spks = Box::new(spks.chain(unused_spks.into_iter().map(|(k, i, spk)| {
eprintln!(
"Checking if address {} {}:{} has been used",
Address::from_script(&spk, args.network).unwrap(),
k,
i,
);
// Flush early to ensure we print at every iteration.
let _ = io::stderr().flush();
spk
})));
request =
request.chain_spks(unused_spks.into_iter().map(move |(k, i, spk)| {
eprint!(
"Checking if address {} {}:{} has been used",
Address::from_script(&spk, args.network).unwrap(),
k,
i,
);
// Flush early to ensure we print at every iteration.
let _ = io::stderr().flush();
spk
}));
}
if utxos {
// We want to search for whether the UTXO is spent, and spent by which
@@ -295,11 +283,11 @@ fn main() -> anyhow::Result<()> {
.filter_chain_unspents(&*chain, local_tip.block_id(), init_outpoints)
.map(|(_, utxo)| utxo)
.collect::<Vec<_>>();
outpoints = Box::new(
request = request.chain_outpoints(
utxos
.into_iter()
.inspect(|utxo| {
eprintln!(
eprint!(
"Checking if outpoint {} (value: {}) has been spent",
utxo.outpoint, utxo.txout.value
);
@@ -319,29 +307,48 @@ fn main() -> anyhow::Result<()> {
.filter(|canonical_tx| !canonical_tx.chain_position.is_confirmed())
.map(|canonical_tx| canonical_tx.tx_node.txid)
.collect::<Vec<Txid>>();
txids = Box::new(unconfirmed_txids.into_iter().inspect(|txid| {
eprintln!("Checking if {} is confirmed yet", txid);
request = request.chain_txids(unconfirmed_txids.into_iter().inspect(|txid| {
eprint!("Checking if {} is confirmed yet", txid);
// Flush early to ensure we print at every iteration.
let _ = io::stderr().flush();
}));
}
}
let mut update = client.sync(
local_tip,
spks,
txids,
outpoints,
scan_options.parallel_requests,
)?;
let total_spks = request.spks.len();
let total_txids = request.txids.len();
let total_ops = request.outpoints.len();
request = request
.inspect_spks({
let mut visited = 0;
move |_| {
visited += 1;
eprintln!(" [ {:>6.2}% ]", (visited * 100) as f32 / total_spks as f32)
}
})
.inspect_txids({
let mut visited = 0;
move |_| {
visited += 1;
eprintln!(" [ {:>6.2}% ]", (visited * 100) as f32 / total_txids as f32)
}
})
.inspect_outpoints({
let mut visited = 0;
move |_| {
visited += 1;
eprintln!(" [ {:>6.2}% ]", (visited * 100) as f32 / total_ops as f32)
}
});
let mut update = client.sync(request, scan_options.parallel_requests)?;
// Update last seen unconfirmed
let now = std::time::UNIX_EPOCH.elapsed().unwrap().as_secs();
let _ = update.tx_graph.update_last_seen_unconfirmed(now);
let _ = update.graph_update.update_last_seen_unconfirmed(now);
(
chain.lock().unwrap().apply_update(update.local_chain)?,
graph.lock().unwrap().apply_update(update.tx_graph),
chain.lock().unwrap().apply_update(update.chain_update)?,
graph.lock().unwrap().apply_update(update.graph_update),
)
}
};

59
example-crates/wallet_esplora_async/src/main.rs
View File

@@ -1,8 +1,7 @@
use std::{io::Write, str::FromStr};
use std::{collections::BTreeSet, io::Write, str::FromStr};
use bdk::{
bitcoin::{Address, Network},
wallet::Update,
bitcoin::{Address, Network, Script},
KeychainKind, SignOptions, Wallet,
};
use bdk_esplora::{esplora_client, EsploraAsyncExt};
@@ -37,34 +36,44 @@ async fn main() -> Result<(), anyhow::Error> {
let client =
esplora_client::Builder::new("https://blockstream.info/testnet/api").build_async()?;
let prev_tip = wallet.latest_checkpoint();
let keychain_spks = wallet
.all_unbounded_spk_iters()
.into_iter()
.map(|(k, k_spks)| {
let mut once = Some(());
let mut stdout = std::io::stdout();
let k_spks = k_spks
.inspect(move |(spk_i, _)| match once.take() {
Some(_) => print!("\nScanning keychain [{:?}]", k),
None => print!(" {:<3}", spk_i),
})
.inspect(move |_| stdout.flush().expect("must flush"));
(k, k_spks)
fn generate_inspect(kind: KeychainKind) -> impl FnMut(u32, &Script) + Send + Sync + 'static {
let mut once = Some(());
let mut stdout = std::io::stdout();
move |spk_i, _| {
match once.take() {
Some(_) => print!("\nScanning keychain [{:?}]", kind),
None => print!(" {:<3}", spk_i),
};
stdout.flush().expect("must flush");
}
}
let request = wallet
.start_full_scan()
.inspect_spks_for_all_keychains({
let mut once = BTreeSet::<KeychainKind>::new();
move |keychain, spk_i, _| {
match once.insert(keychain) {
true => print!("\nScanning keychain [{:?}]", keychain),
false => print!(" {:<3}", spk_i),
}
std::io::stdout().flush().expect("must flush")
}
})
.collect();
.inspect_spks_for_keychain(
KeychainKind::External,
generate_inspect(KeychainKind::External),
)
.inspect_spks_for_keychain(
KeychainKind::Internal,
generate_inspect(KeychainKind::Internal),
);
let mut update = client
.full_scan(prev_tip, keychain_spks, STOP_GAP, PARALLEL_REQUESTS)
.full_scan(request, STOP_GAP, PARALLEL_REQUESTS)
.await?;
let now = std::time::UNIX_EPOCH.elapsed().unwrap().as_secs();
let _ = update.tx_graph.update_last_seen_unconfirmed(now);
let _ = update.graph_update.update_last_seen_unconfirmed(now);
let update = Update {
last_active_indices: update.last_active_indices,
graph: update.tx_graph,
chain: Some(update.local_chain),
};
wallet.apply_update(update)?;
wallet.commit()?;
println!();

43
example-crates/wallet_esplora_blocking/src/main.rs
View File

@@ -3,11 +3,10 @@ const SEND_AMOUNT: u64 = 1000;
const STOP_GAP: usize = 5;
const PARALLEL_REQUESTS: usize = 1;
use std::{io::Write, str::FromStr};
use std::{collections::BTreeSet, io::Write, str::FromStr};
use bdk::{
bitcoin::{Address, Network},
wallet::Update,
KeychainKind, SignOptions, Wallet,
};
use bdk_esplora::{esplora_client, EsploraExt};
@@ -36,36 +35,22 @@ fn main() -> Result<(), anyhow::Error> {
let client =
esplora_client::Builder::new("https://blockstream.info/testnet/api").build_blocking();
let keychain_spks = wallet
.all_unbounded_spk_iters()
.into_iter()
.map(|(k, k_spks)| {
let mut once = Some(());
let mut stdout = std::io::stdout();
let k_spks = k_spks
.inspect(move |(spk_i, _)| match once.take() {
Some(_) => print!("\nScanning keychain [{:?}]", k),
None => print!(" {:<3}", spk_i),
})
.inspect(move |_| stdout.flush().expect("must flush"));
(k, k_spks)
})
.collect();
let request = wallet.start_full_scan().inspect_spks_for_all_keychains({
let mut once = BTreeSet::<KeychainKind>::new();
move |keychain, spk_i, _| {
match once.insert(keychain) {
true => print!("\nScanning keychain [{:?}]", keychain),
false => print!(" {:<3}", spk_i),
};
std::io::stdout().flush().expect("must flush")
}
});
let mut update = client.full_scan(
wallet.latest_checkpoint(),
keychain_spks,
STOP_GAP,
PARALLEL_REQUESTS,
)?;
let mut update = client.full_scan(request, STOP_GAP, PARALLEL_REQUESTS)?;
let now = std::time::UNIX_EPOCH.elapsed().unwrap().as_secs();
let _ = update.tx_graph.update_last_seen_unconfirmed(now);
let _ = update.graph_update.update_last_seen_unconfirmed(now);
wallet.apply_update(Update {
last_active_indices: update.last_active_indices,
graph: update.tx_graph,
chain: Some(update.local_chain),
})?;
wallet.apply_update(update)?;
wallet.commit()?;
println!();