From ea62337f0d0a806c7560bac03799850e8bf76de8 Mon Sep 17 00:00:00 2001 From: Alekos Filini Date: Tue, 30 Jun 2020 15:21:14 +0200 Subject: [PATCH] [database] Replace DerivationPaths with single u32s --- src/blockchain/utils.rs | 8 ++-- src/database/keyvalue.rs | 56 ++++++++++++---------------- src/database/memory.rs | 80 ++++++++++++++++------------------------ src/database/mod.rs | 17 ++++----- src/wallet/mod.rs | 57 ++++++++-------------------- 5 files changed, 81 insertions(+), 137 deletions(-) diff --git a/src/blockchain/utils.rs b/src/blockchain/utils.rs index 326bd1d9..4bbe6281 100644 --- a/src/blockchain/utils.rs +++ b/src/blockchain/utils.rs @@ -226,7 +226,7 @@ pub trait ElectrumLikeSync { let mut to_check_later = vec![]; for (i, output) in tx.output.iter().enumerate() { // this output is ours, we have a path to derive it - if let Some((script_type, path)) = + if let Some((script_type, child)) = database.get_path_from_script_pubkey(&output.script_pubkey)? { debug!("{} output #{} is mine, adding utxo", txid, i); @@ -242,10 +242,8 @@ pub trait ElectrumLikeSync { } // derive as many change addrs as external addresses that we've seen - if script_type == ScriptType::Internal - && u32::from(path.as_ref()[0]) > *change_max_deriv - { - *change_max_deriv = u32::from(path.as_ref()[0]); + if script_type == ScriptType::Internal && child > *change_max_deriv { + *change_max_deriv = child; } } } diff --git a/src/database/keyvalue.rs b/src/database/keyvalue.rs index d4f2e2f2..06c55a10 100644 --- a/src/database/keyvalue.rs +++ b/src/database/keyvalue.rs @@ -1,10 +1,9 @@ -use std::convert::{From, TryInto}; +use std::convert::TryInto; use sled::{Batch, Tree}; use bitcoin::consensus::encode::{deserialize, serialize}; use bitcoin::hash_types::Txid; -use bitcoin::util::bip32::{ChildNumber, DerivationPath}; use bitcoin::{OutPoint, Script, Transaction}; use crate::database::memory::MapKey; @@ -14,15 +13,14 @@ use crate::types::*; macro_rules! impl_batch_operations { ( { $($after_insert:tt)* }, $process_delete:ident ) => { - fn set_script_pubkey>(&mut self, script: &Script, script_type: ScriptType, path: &P) -> Result<(), Error> { - let deriv_path = DerivationPath::from(path.as_ref()); - let key = MapKey::Path((Some(script_type), Some(&deriv_path))).as_map_key(); + fn set_script_pubkey(&mut self, script: &Script, script_type: ScriptType, path: u32) -> Result<(), Error> { + let key = MapKey::Path((Some(script_type), Some(path))).as_map_key(); self.insert(key, serialize(script))$($after_insert)*; let key = MapKey::Script(Some(script)).as_map_key(); let value = json!({ "t": script_type, - "p": deriv_path, + "p": path, }); self.insert(key, serde_json::to_vec(&value)?)$($after_insert)*; @@ -70,16 +68,15 @@ macro_rules! impl_batch_operations { Ok(()) } - fn del_script_pubkey_from_path>(&mut self, script_type: ScriptType, path: &P) -> Result, Error> { - let deriv_path = DerivationPath::from(path.as_ref()); - let key = MapKey::Path((Some(script_type), Some(&deriv_path))).as_map_key(); + fn del_script_pubkey_from_path(&mut self, script_type: ScriptType, path: u32) -> Result, Error> { + let key = MapKey::Path((Some(script_type), Some(path))).as_map_key(); let res = self.remove(key); let res = $process_delete!(res); Ok(res.map_or(Ok(None), |x| Some(deserialize(&x)).transpose())?) } - fn del_path_from_script_pubkey(&mut self, script: &Script) -> Result, Error> { + fn del_path_from_script_pubkey(&mut self, script: &Script) -> Result, Error> { let key = MapKey::Script(Some(script)).as_map_key(); let res = self.remove(key); let res = $process_delete!(res); @@ -245,20 +242,19 @@ impl Database for Tree { .collect() } - fn get_script_pubkey_from_path>( + fn get_script_pubkey_from_path( &self, script_type: ScriptType, - path: &P, + path: u32, ) -> Result, Error> { - let deriv_path = DerivationPath::from(path.as_ref()); - let key = MapKey::Path((Some(script_type), Some(&deriv_path))).as_map_key(); + let key = MapKey::Path((Some(script_type), Some(path))).as_map_key(); Ok(self.get(key)?.map(|b| deserialize(&b)).transpose()?) } fn get_path_from_script_pubkey( &self, script: &Script, - ) -> Result, Error> { + ) -> Result, Error> { let key = MapKey::Script(Some(script)).as_map_key(); self.get(key)? .map(|b| -> Result<_, Error> { @@ -415,14 +411,13 @@ mod test { let script = Script::from( Vec::::from_hex("76a91402306a7c23f3e8010de41e9e591348bb83f11daa88ac").unwrap(), ); - let path = DerivationPath::from_str("m/0/1/2/3").unwrap(); + let path = 42; let script_type = ScriptType::External; - tree.set_script_pubkey(&script, script_type, &path).unwrap(); + tree.set_script_pubkey(&script, script_type, path).unwrap(); assert_eq!( - tree.get_script_pubkey_from_path(script_type, &path) - .unwrap(), + tree.get_script_pubkey_from_path(script_type, path).unwrap(), Some(script.clone()) ); assert_eq!( @@ -439,16 +434,13 @@ mod test { let script = Script::from( Vec::::from_hex("76a91402306a7c23f3e8010de41e9e591348bb83f11daa88ac").unwrap(), ); - let path = DerivationPath::from_str("m/0/1/2/3").unwrap(); + let path = 42; let script_type = ScriptType::External; - batch - .set_script_pubkey(&script, script_type, &path) - .unwrap(); + batch.set_script_pubkey(&script, script_type, path).unwrap(); assert_eq!( - tree.get_script_pubkey_from_path(script_type, &path) - .unwrap(), + tree.get_script_pubkey_from_path(script_type, path).unwrap(), None ); assert_eq!(tree.get_path_from_script_pubkey(&script).unwrap(), None); @@ -456,8 +448,7 @@ mod test { tree.commit_batch(batch).unwrap(); assert_eq!( - tree.get_script_pubkey_from_path(script_type, &path) - .unwrap(), + tree.get_script_pubkey_from_path(script_type, path).unwrap(), Some(script.clone()) ); assert_eq!( @@ -473,10 +464,10 @@ mod test { let script = Script::from( Vec::::from_hex("76a91402306a7c23f3e8010de41e9e591348bb83f11daa88ac").unwrap(), ); - let path = DerivationPath::from_str("m/0/1/2/3").unwrap(); + let path = 42; let script_type = ScriptType::External; - tree.set_script_pubkey(&script, script_type, &path).unwrap(); + tree.set_script_pubkey(&script, script_type, path).unwrap(); assert_eq!(tree.iter_script_pubkeys(None).unwrap().len(), 1); } @@ -488,14 +479,13 @@ mod test { let script = Script::from( Vec::::from_hex("76a91402306a7c23f3e8010de41e9e591348bb83f11daa88ac").unwrap(), ); - let path = DerivationPath::from_str("m/0/1/2/3").unwrap(); + let path = 42; let script_type = ScriptType::External; - tree.set_script_pubkey(&script, script_type, &path).unwrap(); + tree.set_script_pubkey(&script, script_type, path).unwrap(); assert_eq!(tree.iter_script_pubkeys(None).unwrap().len(), 1); - tree.del_script_pubkey_from_path(script_type, &path) - .unwrap(); + tree.del_script_pubkey_from_path(script_type, path).unwrap(); assert_eq!(tree.iter_script_pubkeys(None).unwrap().len(), 0); } diff --git a/src/database/memory.rs b/src/database/memory.rs index 29bc0df1..b17bd549 100644 --- a/src/database/memory.rs +++ b/src/database/memory.rs @@ -3,7 +3,6 @@ use std::ops::Bound::{Excluded, Included}; use bitcoin::consensus::encode::{deserialize, serialize}; use bitcoin::hash_types::Txid; -use bitcoin::util::bip32::{ChildNumber, DerivationPath}; use bitcoin::{OutPoint, Script, Transaction}; use crate::database::{BatchDatabase, BatchOperations, Database}; @@ -19,7 +18,7 @@ use crate::types::*; // descriptor checksum d{i,e} -> vec pub(crate) enum MapKey<'a> { - Path((Option, Option<&'a DerivationPath>)), + Path((Option, Option)), Script(Option<&'a Script>), UTXO(Option<&'a OutPoint>), RawTx(Option<&'a Txid>), @@ -49,13 +48,7 @@ impl MapKey<'_> { fn serialize_content(&self) -> Vec { match self { - MapKey::Path((_, Some(path))) => { - let mut res = vec![]; - for val in *path { - res.extend(&u32::from(*val).to_be_bytes()); - } - res - } + MapKey::Path((_, Some(child))) => u32::from(*child).to_be_bytes().to_vec(), MapKey::Script(Some(s)) => serialize(*s), MapKey::UTXO(Some(s)) => serialize(*s), MapKey::RawTx(Some(s)) => serialize(*s), @@ -99,20 +92,19 @@ impl MemoryDatabase { } impl BatchOperations for MemoryDatabase { - fn set_script_pubkey>( + fn set_script_pubkey( &mut self, script: &Script, script_type: ScriptType, - path: &P, + path: u32, ) -> Result<(), Error> { - let deriv_path = DerivationPath::from(path.as_ref()); - let key = MapKey::Path((Some(script_type), Some(&deriv_path))).as_map_key(); + let key = MapKey::Path((Some(script_type), Some(path))).as_map_key(); self.map.insert(key, Box::new(script.clone())); let key = MapKey::Script(Some(script)).as_map_key(); let value = json!({ "t": script_type, - "p": deriv_path, + "p": path, }); self.map.insert(key, Box::new(value)); @@ -154,13 +146,12 @@ impl BatchOperations for MemoryDatabase { Ok(()) } - fn del_script_pubkey_from_path>( + fn del_script_pubkey_from_path( &mut self, script_type: ScriptType, - path: &P, + path: u32, ) -> Result, Error> { - let deriv_path = DerivationPath::from(path.as_ref()); - let key = MapKey::Path((Some(script_type), Some(&deriv_path))).as_map_key(); + let key = MapKey::Path((Some(script_type), Some(path))).as_map_key(); let res = self.map.remove(&key); self.deleted_keys.push(key); @@ -169,7 +160,7 @@ impl BatchOperations for MemoryDatabase { fn del_path_from_script_pubkey( &mut self, script: &Script, - ) -> Result, Error> { + ) -> Result, Error> { let key = MapKey::Script(Some(script)).as_map_key(); let res = self.map.remove(&key); self.deleted_keys.push(key); @@ -313,13 +304,12 @@ impl Database for MemoryDatabase { .collect() } - fn get_script_pubkey_from_path>( + fn get_script_pubkey_from_path( &self, script_type: ScriptType, - path: &P, + path: u32, ) -> Result, Error> { - let deriv_path = DerivationPath::from(path.as_ref()); - let key = MapKey::Path((Some(script_type), Some(&deriv_path))).as_map_key(); + let key = MapKey::Path((Some(script_type), Some(path))).as_map_key(); Ok(self .map .get(&key) @@ -329,7 +319,7 @@ impl Database for MemoryDatabase { fn get_path_from_script_pubkey( &self, script: &Script, - ) -> Result, Error> { + ) -> Result, Error> { let key = MapKey::Script(Some(script)).as_map_key(); Ok(self.map.get(&key).map(|b| { let mut val: serde_json::Value = b.downcast_ref().cloned().unwrap(); @@ -410,8 +400,6 @@ impl BatchDatabase for MemoryDatabase { #[cfg(test)] mod test { use std::str::FromStr; - use std::sync::{Arc, Condvar, Mutex, Once}; - use std::time::{SystemTime, UNIX_EPOCH}; use bitcoin::consensus::encode::deserialize; use bitcoin::hashes::hex::*; @@ -431,14 +419,13 @@ mod test { let script = Script::from( Vec::::from_hex("76a91402306a7c23f3e8010de41e9e591348bb83f11daa88ac").unwrap(), ); - let path = DerivationPath::from_str("m/0/1/2/3").unwrap(); + let path = 42; let script_type = ScriptType::External; - tree.set_script_pubkey(&script, script_type, &path).unwrap(); + tree.set_script_pubkey(&script, script_type, path).unwrap(); assert_eq!( - tree.get_script_pubkey_from_path(script_type, &path) - .unwrap(), + tree.get_script_pubkey_from_path(script_type, path).unwrap(), Some(script.clone()) ); assert_eq!( @@ -455,16 +442,13 @@ mod test { let script = Script::from( Vec::::from_hex("76a91402306a7c23f3e8010de41e9e591348bb83f11daa88ac").unwrap(), ); - let path = DerivationPath::from_str("m/0/1/2/3").unwrap(); + let path = 42; let script_type = ScriptType::External; - batch - .set_script_pubkey(&script, script_type, &path) - .unwrap(); + batch.set_script_pubkey(&script, script_type, path).unwrap(); assert_eq!( - tree.get_script_pubkey_from_path(script_type, &path) - .unwrap(), + tree.get_script_pubkey_from_path(script_type, path).unwrap(), None ); assert_eq!(tree.get_path_from_script_pubkey(&script).unwrap(), None); @@ -472,13 +456,12 @@ mod test { tree.commit_batch(batch).unwrap(); assert_eq!( - tree.get_script_pubkey_from_path(script_type, &path) - .unwrap(), + tree.get_script_pubkey_from_path(script_type, path).unwrap(), Some(script.clone()) ); assert_eq!( tree.get_path_from_script_pubkey(&script).unwrap(), - Some((script_type, path.clone())) + Some((script_type, path)) ); } @@ -489,10 +472,10 @@ mod test { let script = Script::from( Vec::::from_hex("76a91402306a7c23f3e8010de41e9e591348bb83f11daa88ac").unwrap(), ); - let path = DerivationPath::from_str("m/0/1/2/3").unwrap(); + let path = 42; let script_type = ScriptType::External; - tree.set_script_pubkey(&script, script_type, &path).unwrap(); + tree.set_script_pubkey(&script, script_type, path).unwrap(); assert_eq!(tree.iter_script_pubkeys(None).unwrap().len(), 1); } @@ -504,14 +487,13 @@ mod test { let script = Script::from( Vec::::from_hex("76a91402306a7c23f3e8010de41e9e591348bb83f11daa88ac").unwrap(), ); - let path = DerivationPath::from_str("m/0/1/2/3").unwrap(); + let path = 42; let script_type = ScriptType::External; - tree.set_script_pubkey(&script, script_type, &path).unwrap(); + tree.set_script_pubkey(&script, script_type, path).unwrap(); assert_eq!(tree.iter_script_pubkeys(None).unwrap().len(), 1); - tree.del_script_pubkey_from_path(script_type, &path) - .unwrap(); + tree.del_script_pubkey_from_path(script_type, path).unwrap(); assert_eq!(tree.iter_script_pubkeys(None).unwrap().len(), 0); } @@ -522,20 +504,20 @@ mod test { let script = Script::from( Vec::::from_hex("76a91402306a7c23f3e8010de41e9e591348bb83f11daa88ac").unwrap(), ); - let path = DerivationPath::from_str("m/0/1/2/3").unwrap(); + let path = 42; let script_type = ScriptType::External; - tree.set_script_pubkey(&script, script_type, &path).unwrap(); + tree.set_script_pubkey(&script, script_type, path).unwrap(); assert_eq!(tree.iter_script_pubkeys(None).unwrap().len(), 1); let mut batch = tree.begin_batch(); batch - .del_script_pubkey_from_path(script_type, &path) + .del_script_pubkey_from_path(script_type, path) .unwrap(); assert_eq!(tree.iter_script_pubkeys(None).unwrap().len(), 1); - tree.commit_batch(batch); + tree.commit_batch(batch).unwrap(); assert_eq!(tree.iter_script_pubkeys(None).unwrap().len(), 0); } diff --git a/src/database/mod.rs b/src/database/mod.rs index e4a69234..8fc0c93a 100644 --- a/src/database/mod.rs +++ b/src/database/mod.rs @@ -1,5 +1,4 @@ use bitcoin::hash_types::Txid; -use bitcoin::util::bip32::{ChildNumber, DerivationPath}; use bitcoin::{OutPoint, Script, Transaction, TxOut}; use crate::error::Error; @@ -10,26 +9,26 @@ pub mod keyvalue; pub mod memory; pub trait BatchOperations { - fn set_script_pubkey>( + fn set_script_pubkey( &mut self, script: &Script, script_type: ScriptType, - path: &P, + child: u32, ) -> Result<(), Error>; fn set_utxo(&mut self, utxo: &UTXO) -> Result<(), Error>; fn set_raw_tx(&mut self, transaction: &Transaction) -> Result<(), Error>; fn set_tx(&mut self, transaction: &TransactionDetails) -> Result<(), Error>; fn set_last_index(&mut self, script_type: ScriptType, value: u32) -> Result<(), Error>; - fn del_script_pubkey_from_path>( + fn del_script_pubkey_from_path( &mut self, script_type: ScriptType, - path: &P, + child: u32, ) -> Result, Error>; fn del_path_from_script_pubkey( &mut self, script: &Script, - ) -> Result, Error>; + ) -> Result, Error>; fn del_utxo(&mut self, outpoint: &OutPoint) -> Result, Error>; fn del_raw_tx(&mut self, txid: &Txid) -> Result, Error>; fn del_tx( @@ -52,15 +51,15 @@ pub trait Database: BatchOperations { fn iter_raw_txs(&self) -> Result, Error>; fn iter_txs(&self, include_raw: bool) -> Result, Error>; - fn get_script_pubkey_from_path>( + fn get_script_pubkey_from_path( &self, script_type: ScriptType, - path: &P, + child: u32, ) -> Result, Error>; fn get_path_from_script_pubkey( &self, script: &Script, - ) -> Result, Error>; + ) -> Result, Error>; fn get_utxo(&self, outpoint: &OutPoint) -> Result, Error>; fn get_raw_tx(&self, txid: &Txid) -> Result, Error>; fn get_tx(&self, txid: &Txid, include_raw: bool) -> Result, Error>; diff --git a/src/wallet/mod.rs b/src/wallet/mod.rs index 1a88cdd7..2288010a 100644 --- a/src/wallet/mod.rs +++ b/src/wallet/mod.rs @@ -7,7 +7,6 @@ use std::time::{Instant, SystemTime, UNIX_EPOCH}; use bitcoin::blockdata::opcodes; use bitcoin::blockdata::script::Builder; use bitcoin::consensus::encode::serialize; -use bitcoin::util::bip32::{ChildNumber, DerivationPath}; use bitcoin::util::psbt::PartiallySignedTransaction as PSBT; use bitcoin::{ Address, Network, OutPoint, PublicKey, Script, SigHashType, Transaction, TxIn, TxOut, Txid, @@ -247,22 +246,15 @@ where let mut psbt = PSBT::from_unsigned_tx(tx)?; // add metadata for the inputs - for ((psbt_input, (script_type, path)), input) in psbt + for ((psbt_input, (script_type, child)), input) in psbt .inputs .iter_mut() .zip(paths.into_iter()) .zip(psbt.global.unsigned_tx.input.iter()) { - let path: Vec = path.into(); - let index = match path.last() { - Some(ChildNumber::Normal { index }) => *index, - Some(ChildNumber::Hardened { index }) => *index, - None => 0, - }; - let desc = self.get_descriptor_for(script_type); - psbt_input.hd_keypaths = desc.get_hd_keypaths(index).unwrap(); - let derived_descriptor = desc.derive(index).unwrap(); + psbt_input.hd_keypaths = desc.get_hd_keypaths(child).unwrap(); + let derived_descriptor = desc.derive(child).unwrap(); // TODO: figure out what do redeem_script and witness_script mean psbt_input.redeem_script = derived_descriptor.psbt_redeem_script(); @@ -290,20 +282,13 @@ where .iter_mut() .zip(psbt.global.unsigned_tx.output.iter()) { - if let Some((script_type, derivation_path)) = self + if let Some((script_type, child)) = self .database .borrow() .get_path_from_script_pubkey(&tx_output.script_pubkey)? { - let derivation_path: Vec = derivation_path.into(); - let index = match derivation_path.last() { - Some(ChildNumber::Normal { index }) => *index, - Some(ChildNumber::Hardened { index }) => *index, - None => 0, - }; - let desc = self.get_descriptor_for(script_type); - psbt_output.hd_keypaths = desc.get_hd_keypaths(index)?; + psbt_output.hd_keypaths = desc.get_hd_keypaths(child)?; } } @@ -638,9 +623,9 @@ where outgoing: u64, input_witness_weight: usize, mut fee_val: f32, - ) -> Result<(Vec, Vec<(ScriptType, DerivationPath)>, u64, f32), Error> { + ) -> Result<(Vec, Vec<(ScriptType, u32)>, u64, f32), Error> { let mut answer = Vec::new(); - let mut paths = Vec::new(); + let mut deriv_indexes = Vec::new(); let calc_fee_bytes = |wu| (wu as f32) * fee_rate / 4.0; debug!( @@ -674,15 +659,15 @@ where answer.push(new_in); selected_amount += utxo.txout.value; - let path = self + let child = self .database .borrow() .get_path_from_script_pubkey(&utxo.txout.script_pubkey)? .unwrap(); // TODO: remove unrwap - paths.push(path); + deriv_indexes.push(child); } - Ok((answer, paths, selected_amount, fee_val)) + Ok((answer, deriv_indexes, selected_amount, fee_val)) } fn add_hd_keypaths(&self, psbt: &mut PSBT) -> Result<(), Error> { @@ -703,21 +688,14 @@ where debug!("found descriptor path {:?}", option_path); - let (script_type, path) = match option_path { + let (script_type, child) = match option_path { None => continue, - Some((script_type, path)) => (script_type, path), - }; - - // TODO: this is duplicated code - let index = match path.into_iter().last() { - Some(ChildNumber::Normal { index }) => *index, - Some(ChildNumber::Hardened { index }) => *index, - None => 0, + Some((script_type, child)) => (script_type, child), }; // merge hd_keypaths let desc = self.get_descriptor_for(script_type); - let mut hd_keypaths = desc.get_hd_keypaths(index)?; + let mut hd_keypaths = desc.get_hd_keypaths(child)?; psbt_input.hd_keypaths.append(&mut hd_keypaths); } } @@ -792,11 +770,10 @@ where // TODO: // let batch_query_size = batch_query_size.unwrap_or(20); - let path = DerivationPath::from(vec![ChildNumber::Normal { index: max_address }]); let last_addr = self .database .borrow() - .get_script_pubkey_from_path(ScriptType::External, &path)?; + .get_script_pubkey_from_path(ScriptType::External, max_address)?; // cache a few of our addresses if last_addr.is_none() { @@ -806,23 +783,21 @@ where for i in 0..=max_address { let derived = self.descriptor.derive(i).unwrap(); - let full_path = DerivationPath::from(vec![ChildNumber::Normal { index: i }]); address_batch.set_script_pubkey( &derived.script_pubkey(), ScriptType::External, - &full_path, + i, )?; } if self.change_descriptor.is_some() { for i in 0..=max_address { let derived = self.change_descriptor.as_ref().unwrap().derive(i).unwrap(); - let full_path = DerivationPath::from(vec![ChildNumber::Normal { index: i }]); address_batch.set_script_pubkey( &derived.script_pubkey(), ScriptType::Internal, - &full_path, + i, )?; } }