From c06d9f1d33563194e91ba19a190dc12e8f11f1a6 Mon Sep 17 00:00:00 2001 From: John Cantrell Date: Fri, 18 Jun 2021 13:45:16 -0400 Subject: [PATCH] implement sqlite database --- .github/workflows/cont_integration.yml | 1 + .github/workflows/nightly_docs.yml | 2 +- CHANGELOG.md | 1 + Cargo.toml | 2 + src/database/any.rs | 50 +- src/database/mod.rs | 5 + src/database/sqlite.rs | 968 +++++++++++++++++++++++++ src/error.rs | 5 + src/lib.rs | 3 + 9 files changed, 1033 insertions(+), 4 deletions(-) create mode 100644 src/database/sqlite.rs diff --git a/.github/workflows/cont_integration.yml b/.github/workflows/cont_integration.yml index 233c98ee..992b7ab3 100644 --- a/.github/workflows/cont_integration.yml +++ b/.github/workflows/cont_integration.yml @@ -26,6 +26,7 @@ jobs: - verify - async-interface - use-esplora-reqwest + - sqlite steps: - name: checkout uses: actions/checkout@v2 diff --git a/.github/workflows/nightly_docs.yml b/.github/workflows/nightly_docs.yml index e6a49e2e..88f72ba1 100644 --- a/.github/workflows/nightly_docs.yml +++ b/.github/workflows/nightly_docs.yml @@ -24,7 +24,7 @@ jobs: - name: Update toolchain run: rustup update - name: Build docs - run: cargo rustdoc --verbose --features=compiler,electrum,esplora,ureq,compact_filters,key-value-db,all-keys -- --cfg docsrs -Dwarnings + run: cargo rustdoc --verbose --features=compiler,electrum,esplora,ureq,compact_filters,key-value-db,all-keys,sqlite -- --cfg docsrs -Dwarnings - name: Upload artifact uses: actions/upload-artifact@v2 with: diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a4b347c..6ab9af73 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added `flush` method to the `Database` trait to explicitly flush to disk latest changes on the db. - Add support for proxies in `EsploraBlockchain` +- Added `SqliteDatabase` that implements `Database` backed by a sqlite database using `rusqlite` crate. ## [v0.10.0] - [v0.9.0] diff --git a/Cargo.toml b/Cargo.toml index b2b248bd..ce4e40d5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,6 +23,7 @@ rand = "^0.7" # Optional dependencies sled = { version = "0.34", optional = true } electrum-client = { version = "0.8", optional = true } +rusqlite = { version = "0.25.3", optional = true } reqwest = { version = "0.11", optional = true, features = ["json"] } ureq = { version = "2.1", features = ["json"], optional = true } futures = { version = "0.3", optional = true } @@ -55,6 +56,7 @@ minimal = [] compiler = ["miniscript/compiler"] verify = ["bitcoinconsensus"] default = ["key-value-db", "electrum"] +sqlite = ["rusqlite"] compact_filters = ["rocksdb", "socks", "lazy_static", "cc"] key-value-db = ["sled"] all-keys = ["keys-bip39"] diff --git a/src/database/any.rs b/src/database/any.rs index dbdd2d09..707d40fc 100644 --- a/src/database/any.rs +++ b/src/database/any.rs @@ -65,6 +65,8 @@ macro_rules! impl_inner_method { $enum_name::Memory(inner) => inner.$name( $($args, )* ), #[cfg(feature = "key-value-db")] $enum_name::Sled(inner) => inner.$name( $($args, )* ), + #[cfg(feature = "sqlite")] + $enum_name::Sqlite(inner) => inner.$name( $($args, )* ), } } } @@ -82,10 +84,15 @@ pub enum AnyDatabase { #[cfg_attr(docsrs, doc(cfg(feature = "key-value-db")))] /// Simple key-value embedded database based on [`sled`] Sled(sled::Tree), + #[cfg(feature = "sqlite")] + #[cfg_attr(docsrs, doc(cfg(feature = "sqlite")))] + /// Sqlite embedded database using [`rusqlite`] + Sqlite(sqlite::SqliteDatabase), } impl_from!(memory::MemoryDatabase, AnyDatabase, Memory,); impl_from!(sled::Tree, AnyDatabase, Sled, #[cfg(feature = "key-value-db")]); +impl_from!(sqlite::SqliteDatabase, AnyDatabase, Sqlite, #[cfg(feature = "sqlite")]); /// Type that contains any of the [`BatchDatabase::Batch`] types defined by the library pub enum AnyBatch { @@ -95,6 +102,10 @@ pub enum AnyBatch { #[cfg_attr(docsrs, doc(cfg(feature = "key-value-db")))] /// Simple key-value embedded database based on [`sled`] Sled(::Batch), + #[cfg(feature = "sqlite")] + #[cfg_attr(docsrs, doc(cfg(feature = "sqlite")))] + /// Sqlite embedded database using [`rusqlite`] + Sqlite(::Batch), } impl_from!( @@ -103,6 +114,7 @@ impl_from!( Memory, ); impl_from!(::Batch, AnyBatch, Sled, #[cfg(feature = "key-value-db")]); +impl_from!(::Batch, AnyBatch, Sqlite, #[cfg(feature = "sqlite")]); impl BatchOperations for AnyDatabase { fn set_script_pubkey( @@ -300,19 +312,25 @@ impl BatchDatabase for AnyDatabase { AnyDatabase::Memory(inner) => inner.begin_batch().into(), #[cfg(feature = "key-value-db")] AnyDatabase::Sled(inner) => inner.begin_batch().into(), + #[cfg(feature = "sqlite")] + AnyDatabase::Sqlite(inner) => inner.begin_batch().into(), } } fn commit_batch(&mut self, batch: Self::Batch) -> Result<(), Error> { match self { AnyDatabase::Memory(db) => match batch { AnyBatch::Memory(batch) => db.commit_batch(batch), - #[cfg(feature = "key-value-db")] - _ => unimplemented!("Sled batch shouldn't be used with Memory db."), + _ => unimplemented!("Other batch shouldn't be used with Memory db."), }, #[cfg(feature = "key-value-db")] AnyDatabase::Sled(db) => match batch { AnyBatch::Sled(batch) => db.commit_batch(batch), - _ => unimplemented!("Memory batch shouldn't be used with Sled db."), + _ => unimplemented!("Other batch shouldn't be used with Sled db."), + }, + #[cfg(feature = "sqlite")] + AnyDatabase::Sqlite(db) => match batch { + AnyBatch::Sqlite(batch) => db.commit_batch(batch), + _ => unimplemented!("Other batch shouldn't be used with Sqlite db."), }, } } @@ -337,6 +355,23 @@ impl ConfigurableDatabase for sled::Tree { } } +/// Configuration type for a [`sqlite::SqliteDatabase`] database +#[cfg(feature = "sqlite")] +#[derive(Debug, serde::Serialize, serde::Deserialize)] +pub struct SqliteDbConfiguration { + /// Main directory of the db + pub path: String, +} + +#[cfg(feature = "sqlite")] +impl ConfigurableDatabase for sqlite::SqliteDatabase { + type Config = SqliteDbConfiguration; + + fn from_config(config: &Self::Config) -> Result { + Ok(sqlite::SqliteDatabase::new(config.path.clone())) + } +} + /// Type that can contain any of the database configurations defined by the library /// /// This allows storing a single configuration that can be loaded into an [`AnyDatabase`] @@ -350,6 +385,10 @@ pub enum AnyDatabaseConfig { #[cfg_attr(docsrs, doc(cfg(feature = "key-value-db")))] /// Simple key-value embedded database based on [`sled`] Sled(SledDbConfiguration), + #[cfg(feature = "sqlite")] + #[cfg_attr(docsrs, doc(cfg(feature = "sqlite")))] + /// Sqlite embedded database using [`rusqlite`] + Sqlite(SqliteDbConfiguration), } impl ConfigurableDatabase for AnyDatabase { @@ -362,9 +401,14 @@ impl ConfigurableDatabase for AnyDatabase { } #[cfg(feature = "key-value-db")] AnyDatabaseConfig::Sled(inner) => AnyDatabase::Sled(sled::Tree::from_config(inner)?), + #[cfg(feature = "sqlite")] + AnyDatabaseConfig::Sqlite(inner) => { + AnyDatabase::Sqlite(sqlite::SqliteDatabase::from_config(inner)?) + } }) } } impl_from!((), AnyDatabaseConfig, Memory,); impl_from!(SledDbConfiguration, AnyDatabaseConfig, Sled, #[cfg(feature = "key-value-db")]); +impl_from!(SqliteDbConfiguration, AnyDatabaseConfig, Sqlite, #[cfg(feature = "sqlite")]); diff --git a/src/database/mod.rs b/src/database/mod.rs index 6dbecc66..4a3936f5 100644 --- a/src/database/mod.rs +++ b/src/database/mod.rs @@ -36,6 +36,11 @@ pub use any::{AnyDatabase, AnyDatabaseConfig}; #[cfg(feature = "key-value-db")] pub(crate) mod keyvalue; +#[cfg(feature = "sqlite")] +pub(crate) mod sqlite; +#[cfg(feature = "sqlite")] +pub use sqlite::SqliteDatabase; + pub mod memory; pub use memory::MemoryDatabase; diff --git a/src/database/sqlite.rs b/src/database/sqlite.rs new file mode 100644 index 00000000..e396a0a0 --- /dev/null +++ b/src/database/sqlite.rs @@ -0,0 +1,968 @@ +// Bitcoin Dev Kit +// Written in 2020 by Alekos Filini +// +// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers +// +// This file is licensed under the Apache License, Version 2.0 or the MIT license +// , at your option. +// You may not use this file except in accordance with one or both of these +// licenses. + +use bitcoin::consensus::encode::{deserialize, serialize}; +use bitcoin::hash_types::Txid; +use bitcoin::{OutPoint, Script, Transaction, TxOut}; + +use crate::database::{BatchDatabase, BatchOperations, Database}; +use crate::error::Error; +use crate::types::*; + +use rusqlite::{named_params, Connection}; + +static MIGRATIONS: &[&str] = &[ + "CREATE TABLE version (version INTEGER)", + "INSERT INTO version VALUES (1)", + "CREATE TABLE script_pubkeys (keychain TEXT, child INTEGER, script BLOB);", + "CREATE INDEX idx_keychain_child ON script_pubkeys(keychain, child);", + "CREATE INDEX idx_script ON script_pubkeys(script);", + "CREATE TABLE utxos (value INTEGER, keychain TEXT, vout INTEGER, txid BLOB, script BLOB);", + "CREATE INDEX idx_txid_vout ON utxos(txid, vout);", + "CREATE TABLE transactions (txid BLOB, raw_tx BLOB);", + "CREATE INDEX idx_txid ON transactions(txid);", + "CREATE TABLE transaction_details (txid BLOB, timestamp INTEGER, received INTEGER, sent INTEGER, fee INTEGER, height INTEGER, verified INTEGER DEFAULT 0);", + "CREATE INDEX idx_txdetails_txid ON transaction_details(txid);", + "CREATE TABLE last_derivation_indices (keychain TEXT, value INTEGER);", + "CREATE UNIQUE INDEX idx_indices_keychain ON last_derivation_indices(keychain);", + "CREATE TABLE checksums (keychain TEXT, checksum BLOB);", + "CREATE INDEX idx_checksums_keychain ON checksums(keychain);", +]; + +/// Sqlite database stored on filesystem +/// +/// This is a permanent storage solution for devices and platforms that provide a filesystem. +/// [`crate::database`] +#[derive(Debug)] +pub struct SqliteDatabase { + /// Path on the local filesystem to store the sqlite file + pub path: String, + /// A rusqlite connection object to the sqlite database + pub connection: Connection, +} + +impl SqliteDatabase { + /// Instantiate a new SqliteDatabase instance by creating a connection + /// to the database stored at path + pub fn new(path: String) -> Self { + let connection = get_connection(&path).unwrap(); + SqliteDatabase { path, connection } + } + fn insert_script_pubkey( + &self, + keychain: String, + child: u32, + script: &[u8], + ) -> Result { + let mut statement = self.connection.prepare_cached("INSERT INTO script_pubkeys (keychain, child, script) VALUES (:keychain, :child, :script)")?; + statement.execute(named_params! { + ":keychain": keychain, + ":child": child, + ":script": script + })?; + + Ok(self.connection.last_insert_rowid()) + } + fn insert_utxo( + &self, + value: u64, + keychain: String, + vout: u32, + txid: &[u8], + script: &[u8], + ) -> Result { + let mut statement = self.connection.prepare_cached("INSERT INTO utxos (value, keychain, vout, txid, script) VALUES (:value, :keychain, :vout, :txid, :script)")?; + statement.execute(named_params! { + ":value": value, + ":keychain": keychain, + ":vout": vout, + ":txid": txid, + ":script": script + })?; + + Ok(self.connection.last_insert_rowid()) + } + fn insert_transaction(&self, txid: &[u8], raw_tx: &[u8]) -> Result { + let mut statement = self + .connection + .prepare_cached("INSERT INTO transactions (txid, raw_tx) VALUES (:txid, :raw_tx)")?; + statement.execute(named_params! { + ":txid": txid, + ":raw_tx": raw_tx, + })?; + + Ok(self.connection.last_insert_rowid()) + } + + fn update_transaction(&self, txid: &[u8], raw_tx: &[u8]) -> Result<(), Error> { + let mut statement = self + .connection + .prepare_cached("UPDATE transactions SET raw_tx=:raw_tx WHERE txid=:txid")?; + + statement.execute(named_params! { + ":txid": txid, + ":raw_tx": raw_tx, + })?; + + Ok(()) + } + + fn insert_transaction_details(&self, transaction: &TransactionDetails) -> Result { + let (timestamp, height) = match &transaction.confirmation_time { + Some(confirmation_time) => ( + Some(confirmation_time.timestamp), + Some(confirmation_time.height), + ), + None => (None, None), + }; + + let txid: &[u8] = &transaction.txid; + + let mut statement = self.connection.prepare_cached("INSERT INTO transaction_details (txid, timestamp, received, sent, fee, height, verified) VALUES (:txid, :timestamp, :received, :sent, :fee, :height, :verified)")?; + + statement.execute(named_params! { + ":txid": txid, + ":timestamp": timestamp, + ":received": transaction.received, + ":sent": transaction.sent, + ":fee": transaction.fee, + ":height": height, + ":verified": transaction.verified + })?; + + Ok(self.connection.last_insert_rowid()) + } + + fn update_transaction_details(&self, transaction: &TransactionDetails) -> Result<(), Error> { + let (timestamp, height) = match &transaction.confirmation_time { + Some(confirmation_time) => ( + Some(confirmation_time.timestamp), + Some(confirmation_time.height), + ), + None => (None, None), + }; + + let txid: &[u8] = &transaction.txid; + + let mut statement = self.connection.prepare_cached("UPDATE transaction_details SET timestamp=:timestamp, received=:received, sent=:sent, fee=:fee, height=:height, verified=:verified WHERE txid=:txid")?; + + statement.execute(named_params! { + ":txid": txid, + ":timestamp": timestamp, + ":received": transaction.received, + ":sent": transaction.sent, + ":fee": transaction.fee, + ":height": height, + ":verified": transaction.verified, + })?; + + Ok(()) + } + + fn insert_last_derivation_index(&self, keychain: String, value: u32) -> Result { + let mut statement = self.connection.prepare_cached( + "INSERT INTO last_derivation_indices (keychain, value) VALUES (:keychain, :value)", + )?; + + statement.execute(named_params! { + ":keychain": keychain, + ":value": value, + })?; + + Ok(self.connection.last_insert_rowid()) + } + + fn insert_checksum(&self, keychain: String, checksum: &[u8]) -> Result { + let mut statement = self.connection.prepare_cached( + "INSERT INTO checksums (keychain, checksum) VALUES (:keychain, :checksum)", + )?; + statement.execute(named_params! { + ":keychain": keychain, + ":checksum": checksum, + })?; + + Ok(self.connection.last_insert_rowid()) + } + + fn update_last_derivation_index(&self, keychain: String, value: u32) -> Result<(), Error> { + let mut statement = self.connection.prepare_cached( + "INSERT INTO last_derivation_indices (keychain, value) VALUES (:keychain, :value) ON CONFLICT(keychain) DO UPDATE SET value=:value WHERE keychain=:keychain", + )?; + + statement.execute(named_params! { + ":keychain": keychain, + ":value": value, + })?; + + Ok(()) + } + + fn select_script_pubkeys(&self) -> Result, Error> { + let mut statement = self + .connection + .prepare_cached("SELECT script FROM script_pubkeys")?; + let mut scripts: Vec