Make the blockchain interface async again on wasm32-unknown-unknown

The procedural macro `#[maybe_async]` makes a method or every method of a trait
"async" whenever the target_arch is `wasm32`, and leaves them untouched on
every other platform.

The macro `maybe_await!($e:expr)` can be used to call `maybe_async` methods on
multi-platform code: it expands to `$e` on non-wasm32 platforms and to
`$e.await` on wasm32.

The macro `await_or_block!($e:expr)` can be used to contain async code as much
as possible: it expands to `tokio::runtime::Runtime::new().unwrap().block_on($e)`
on non-wasm32 platforms, and to `$e.await` on wasm32.

.travis.yml

@@ -1,10 +1,15 @@
 language: rust
 rust:
   - stable
-#  - 1.31.0
-#  - 1.22.0
 before_script:
+  # Install a recent version of clang that supports wasm32
+  - wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
+  - sudo apt-add-repository "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-10 main"
+  - sudo apt-get update
+  - sudo apt-get install -y clang-10 libc6-dev-i386
+  # Install the required components and targets
   - rustup component add rustfmt
+  - rustup target add wasm32-unknown-unknown
 script:
   - cargo fmt -- --check --verbose
   - cargo test --verbose --all
@@ -13,6 +18,7 @@ script:
   - cargo build --verbose --no-default-features --features=minimal,esplora
   - cargo build --verbose --no-default-features --features=key-value-db
   - cargo build --verbose --no-default-features --features=electrum
+  - CC="clang-10" CFLAGS="-I/usr/include" cargo build --verbose --no-default-features --features=cli-utils,esplora --target=wasm32-unknown-unknown
 
 notifications:
   email: false

Cargo.toml

@@ -5,6 +5,7 @@ edition = "2018"
 authors = ["Riccardo Casatta <riccardo@casatta.it>", "Alekos Filini <alekos.filini@gmail.com>"]
 
 [dependencies]
+magical-macros = { path = "./macros" }
 log = "^0.4"
 bitcoin = { version = "0.23", features = ["use-serde"] }
 miniscript = { version = "1.0" }
@@ -15,17 +16,23 @@ serde_json = { version = "^1.0" }
 sled = { version = "0.31.0", optional = true }
 electrum-client = { version = "0.2.0-beta.1", optional = true }
 reqwest = { version = "0.10", optional = true, features = ["json"] }
-tokio = { version = "0.2", optional = true, features = ["rt-core"] }
 futures = { version = "0.3", optional = true }
 clap = { version = "2.33", optional = true }
 base64 = { version = "^0.11", optional = true }
 
+# Platform-specific dependencies
+[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
+tokio = { version = "0.2", features = ["rt-core"] }
+
+[target.'cfg(target_arch = "wasm32")'.dependencies]
+async-trait = "0.1"
+
 [features]
 minimal = []
 compiler = ["miniscript/compiler"]
 default = ["key-value-db", "electrum"]
 electrum = ["electrum-client"]
-esplora = ["reqwest", "futures", "tokio"]
+esplora = ["reqwest", "futures"]
 key-value-db = ["sled"]
 cli-utils = ["clap", "base64"]
 

macros/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "magical-macros"
+version = "0.1.0"
+authors = ["Alekos Filini <alekos.filini@gmail.com>"]
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+syn = { version = "1.0", features = ["parsing"] }
+proc-macro2 = "1.0"
+quote = "1.0"
+
+[lib]
+proc-macro = true

macros/src/lib.rs

@@ -0,0 +1,134 @@
+#[macro_use]
+extern crate quote;
+
+use proc_macro::TokenStream;
+
+use syn::spanned::Spanned;
+use syn::{parse, ImplItemMethod, ItemImpl, ItemTrait, Token};
+
+fn add_async_trait(mut parsed: ItemTrait) -> TokenStream {
+    let output = quote! {
+        #[cfg(not(target_arch = "wasm32"))]
+        #parsed
+    };
+
+    for mut item in &mut parsed.items {
+        if let syn::TraitItem::Method(m) = &mut item {
+            m.sig.asyncness = Some(Token![async](m.span()));
+        }
+    }
+
+    let output = quote! {
+        #output
+
+        #[cfg(target_arch = "wasm32")]
+        #[async_trait(?Send)]
+        #parsed
+    };
+
+    output.into()
+}
+
+fn add_async_method(mut parsed: ImplItemMethod) -> TokenStream {
+    let output = quote! {
+        #[cfg(not(target_arch = "wasm32"))]
+        #parsed
+    };
+
+    parsed.sig.asyncness = Some(Token![async](parsed.span()));
+
+    let output = quote! {
+        #output
+
+        #[cfg(target_arch = "wasm32")]
+        #parsed
+    };
+
+    output.into()
+}
+
+fn add_async_impl_trait(mut parsed: ItemImpl) -> TokenStream {
+    let output = quote! {
+        #[cfg(not(target_arch = "wasm32"))]
+        #parsed
+    };
+
+    for mut item in &mut parsed.items {
+        if let syn::ImplItem::Method(m) = &mut item {
+            m.sig.asyncness = Some(Token![async](m.span()));
+        }
+    }
+
+    let output = quote! {
+        #output
+
+        #[cfg(target_arch = "wasm32")]
+        #[async_trait(?Send)]
+        #parsed
+    };
+
+    output.into()
+}
+
+/// Makes a method or every method of a trait "async" only if the target_arch is "wasm32"
+///
+/// Requires the `async-trait` crate as a dependency whenever this attribute is used on a trait
+/// definition or trait implementation.
+#[proc_macro_attribute]
+pub fn maybe_async(_attr: TokenStream, item: TokenStream) -> TokenStream {
+    if let Ok(parsed) = parse(item.clone()) {
+        add_async_trait(parsed)
+    } else if let Ok(parsed) = parse(item.clone()) {
+        add_async_method(parsed)
+    } else if let Ok(parsed) = parse(item) {
+        add_async_impl_trait(parsed)
+    } else {
+        (quote! {
+            compile_error!("#[maybe_async] can only be used on methods, trait or trait impl blocks")
+        }).into()
+    }
+}
+
+/// Awaits if target_arch is "wasm32", does nothing otherwise
+#[proc_macro]
+pub fn maybe_await(expr: TokenStream) -> TokenStream {
+    let expr: proc_macro2::TokenStream = expr.into();
+    let quoted = quote! {
+        {
+            #[cfg(not(target_arch = "wasm32"))]
+            {
+                #expr
+            }
+
+            #[cfg(target_arch = "wasm32")]
+            {
+                #expr.await
+            }
+        }
+    };
+
+    quoted.into()
+}
+
+/// Awaits if target_arch is "wasm32", uses `futures::executor::block_on()` otherwise
+///
+/// Requires the `tokio` crate as a dependecy with `rt-core` or `rt-threaded` to build on non-wasm32 platforms.
+#[proc_macro]
+pub fn await_or_block(expr: TokenStream) -> TokenStream {
+    let expr: proc_macro2::TokenStream = expr.into();
+    let quoted = quote! {
+        {
+            #[cfg(not(target_arch = "wasm32"))]
+            {
+                tokio::runtime::Runtime::new().unwrap().block_on(#expr)
+            }
+
+            #[cfg(target_arch = "wasm32")]
+            {
+                #expr.await
+            }
+        }
+    };
+
+    quoted.into()
+}

src/blockchain/esplora.rs

@@ -1,10 +1,7 @@
 use std::collections::HashSet;
-use std::sync::Mutex;
 
 use futures::stream::{self, StreamExt, TryStreamExt};
 
-use tokio::runtime::Runtime;
-
 #[allow(unused_imports)]
 use log::{debug, error, info, trace};
 
@@ -26,11 +23,8 @@ use crate::error::Error;
 pub struct UrlClient {
     url: String,
     // We use the async client instead of the blocking one because it automatically uses `fetch`
-    // when the target platform is wasm32. For some reason the blocking client doesn't, so we are
+    // when the target platform is wasm32.
-    // stuck with this
     client: Client,
-
-    runtime: Mutex<Runtime>,
 }
 
 #[derive(Debug)]
@@ -47,8 +41,6 @@ impl EsploraBlockchain {
         EsploraBlockchain(Some(UrlClient {
             url: base_url.to_string(),
             client: Client::new(),
-
-            runtime: Mutex::new(Runtime::new().unwrap()),
         }))
     }
 }
@@ -63,6 +55,7 @@ impl Blockchain for EsploraBlockchain {
     }
 }
 
+#[maybe_async]
 impl OnlineBlockchain for EsploraBlockchain {
     fn get_capabilities(&self) -> HashSet<Capability> {
         vec![Capability::FullHistory, Capability::GetAnyTx]
@@ -76,26 +69,35 @@ impl OnlineBlockchain for EsploraBlockchain {
         database: &mut D,
         progress_update: P,
     ) -> Result<(), Error> {
-        self.0
+        maybe_await!(self
-            .as_mut()
-            .ok_or(Error::OfflineClient)?
-            .electrum_like_setup(stop_gap, database, progress_update)
-    }
-
-    fn get_tx(&mut self, txid: &Txid) -> Result<Option<Transaction>, Error> {
-        Ok(self.0.as_mut().ok_or(Error::OfflineClient)?._get_tx(txid)?)
-    }
-
-    fn broadcast(&mut self, tx: &Transaction) -> Result<(), Error> {
-        Ok(self
             .0
             .as_mut()
             .ok_or(Error::OfflineClient)?
-            ._broadcast(tx)?)
+            .electrum_like_setup(stop_gap, database, progress_update))
+    }
+
+    fn get_tx(&mut self, txid: &Txid) -> Result<Option<Transaction>, Error> {
+        Ok(await_or_block!(self
+            .0
+            .as_mut()
+            .ok_or(Error::OfflineClient)?
+            ._get_tx(txid))?)
+    }
+
+    fn broadcast(&mut self, tx: &Transaction) -> Result<(), Error> {
+        Ok(await_or_block!(self
+            .0
+            .as_mut()
+            .ok_or(Error::OfflineClient)?
+            ._broadcast(tx))?)
     }
 
     fn get_height(&mut self) -> Result<usize, Error> {
-        Ok(self.0.as_mut().ok_or(Error::OfflineClient)?._get_height()?)
+        Ok(await_or_block!(self
+            .0
+            .as_mut()
+            .ok_or(Error::OfflineClient)?
+            ._get_height())?)
     }
 }
 
@@ -104,54 +106,39 @@ impl UrlClient {
         sha256::Hash::hash(script.as_bytes()).into_inner().to_hex()
     }
 
-    fn _get_tx(&self, txid: &Txid) -> Result<Option<Transaction>, EsploraError> {
+    async fn _get_tx(&self, txid: &Txid) -> Result<Option<Transaction>, EsploraError> {
-        let resp = self.runtime.lock().unwrap().block_on(
+        let resp = self
-            self.client
+            .client
-                .get(&format!("{}/api/tx/{}/raw", self.url, txid))
+            .get(&format!("{}/api/tx/{}/raw", self.url, txid))
-                .send(),
+            .send()
-        )?;
+            .await?;
 
         if let StatusCode::NOT_FOUND = resp.status() {
             return Ok(None);
         }
 
-        Ok(Some(deserialize(
+        Ok(Some(deserialize(&resp.error_for_status()?.bytes().await?)?))
-            &self
-                .runtime
-                .lock()
-                .unwrap()
-                .block_on(resp.error_for_status()?.bytes())?,
-        )?))
     }
 
-    fn _broadcast(&self, transaction: &Transaction) -> Result<(), EsploraError> {
+    async fn _broadcast(&self, transaction: &Transaction) -> Result<(), EsploraError> {
-        self.runtime
+        self.client
-            .lock()
+            .post(&format!("{}/api/tx", self.url))
-            .unwrap()
+            .body(serialize(transaction).to_hex())
-            .block_on(
+            .send()
-                self.client
+            .await?
-                    .post(&format!("{}/api/tx", self.url))
-                    .body(serialize(transaction).to_hex())
-                    .send(),
-            )?
             .error_for_status()?;
 
         Ok(())
     }
 
-    fn _get_height(&self) -> Result<usize, EsploraError> {
+    async fn _get_height(&self) -> Result<usize, EsploraError> {
-        let req = self.runtime.lock().unwrap().block_on(
+        let req = self
-            self.client
+            .client
-                .get(&format!("{}/api/blocks/tip/height", self.url))
+            .get(&format!("{}/api/blocks/tip/height", self.url))
-                .send(),
+            .send()
-        )?;
+            .await?;
 
-        Ok(self
+        Ok(req.error_for_status()?.text().await?.parse()?)
-            .runtime
-            .lock()
-            .unwrap()
-            .block_on(req.error_for_status()?.text())?
-            .parse()?)
     }
 
     async fn _script_get_history(
@@ -247,34 +234,38 @@ impl UrlClient {
     }
 }
 
+#[maybe_async]
 impl ElectrumLikeSync for UrlClient {
     fn els_batch_script_get_history<'s, I: IntoIterator<Item = &'s Script>>(
         &mut self,
         scripts: I,
     ) -> Result<Vec<Vec<ELSGetHistoryRes>>, Error> {
-        self.runtime.lock().unwrap().block_on(async {
+        let future = async {
             Ok(stream::iter(scripts)
                 .then(|script| self._script_get_history(&script))
                 .try_collect()
                 .await?)
-        })
+        };
+
+        await_or_block!(future)
     }
 
     fn els_batch_script_list_unspent<'s, I: IntoIterator<Item = &'s Script>>(
         &mut self,
         scripts: I,
     ) -> Result<Vec<Vec<ELSListUnspentRes>>, Error> {
-        self.runtime.lock().unwrap().block_on(async {
+        let future = async {
             Ok(stream::iter(scripts)
                 .then(|script| self._script_list_unspent(&script))
                 .try_collect()
                 .await?)
-        })
+        };
+
+        await_or_block!(future)
     }
 
     fn els_transaction_get(&mut self, txid: &Txid) -> Result<Transaction, Error> {
-        Ok(self
+        Ok(await_or_block!(self._get_tx(txid))?
-            ._get_tx(txid)?
             .ok_or_else(|| EsploraError::TransactionNotFound(*txid))?)
     }
 }

src/blockchain/mod.rs

@@ -41,6 +41,7 @@ impl Blockchain for OfflineBlockchain {
     }
 }
 
+#[maybe_async]
 pub trait OnlineBlockchain: Blockchain {
     fn get_capabilities(&self) -> HashSet<Capability>;
 
@@ -56,7 +57,7 @@ pub trait OnlineBlockchain: Blockchain {
         database: &mut D,
         progress_update: P,
     ) -> Result<(), Error> {
-        self.setup(stop_gap, database, progress_update)
+        maybe_await!(self.setup(stop_gap, database, progress_update))
     }
 
     fn get_tx(&mut self, txid: &Txid) -> Result<Option<Transaction>, Error>;

src/blockchain/utils.rs

@@ -27,6 +27,7 @@ pub struct ELSListUnspentRes {
 }
 
 /// Implements the synchronization logic for an Electrum-like client.
+#[maybe_async]
 pub trait ElectrumLikeSync {
     fn els_batch_script_get_history<'s, I: IntoIterator<Item = &'s Script>>(
         &mut self,
@@ -85,7 +86,7 @@ pub trait ElectrumLikeSync {
 
             let until = cmp::min(to_check_later.len(), batch_query_size);
             let chunk: Vec<Script> = to_check_later.drain(..until).collect();
-            let call_result = self.els_batch_script_get_history(chunk.iter())?;
+            let call_result = maybe_await!(self.els_batch_script_get_history(chunk.iter()))?;
 
             for (script, history) in chunk.into_iter().zip(call_result.into_iter()) {
                 trace!("received history for {:?}, size {}", script, history.len());
@@ -93,11 +94,15 @@ pub trait ElectrumLikeSync {
                 if !history.is_empty() {
                     last_found = index;
 
-                    let mut check_later_scripts = self
+                    let mut check_later_scripts = maybe_await!(self.check_history(
-                        .check_history(database, script, history, &mut change_max_deriv)?
+                        database,
-                        .into_iter()
+                        script,
-                        .filter(|x| already_checked.insert(x.clone()))
+                        history,
-                        .collect();
+                        &mut change_max_deriv
+                    ))?
+                    .into_iter()
+                    .filter(|x| already_checked.insert(x.clone()))
+                    .collect();
                     to_check_later.append(&mut check_later_scripts);
                 }
 
@@ -124,7 +129,7 @@ pub trait ElectrumLikeSync {
         let mut batch = database.begin_batch();
         for chunk in ChunksIterator::new(database.iter_utxos()?.into_iter(), batch_query_size) {
             let scripts: Vec<_> = chunk.iter().map(|u| &u.txout.script_pubkey).collect();
-            let call_result = self.els_batch_script_list_unspent(scripts)?;
+            let call_result = maybe_await!(self.els_batch_script_list_unspent(scripts))?;
 
             // check which utxos are actually still unspent
             for (utxo, list_unspent) in chunk.into_iter().zip(call_result.iter()) {
@@ -199,7 +204,7 @@ pub trait ElectrumLikeSync {
                 // went wrong
                 saved_tx.transaction.unwrap()
             }
-            None => self.els_transaction_get(&txid)?,
+            None => maybe_await!(self.els_transaction_get(&txid))?,
         };
 
         let mut incoming: u64 = 0;
@@ -284,13 +289,13 @@ pub trait ElectrumLikeSync {
                 x => u32::try_from(x).ok(),
             };
 
-            to_check_later.extend_from_slice(&self.check_tx_and_descendant(
+            to_check_later.extend_from_slice(&maybe_await!(self.check_tx_and_descendant(
                 database,
                 &tx.tx_hash,
                 height,
                 &script_pubkey,
                 change_max_deriv,
-            )?);
+            ))?);
         }
 
         Ok(to_check_later)

src/cli.rs

@@ -276,6 +276,7 @@ pub fn add_global_flags<'a, 'b>(app: App<'a, 'b>) -> App<'a, 'b> {
     .subcommand(SubCommand::with_name("repl").about("Opens an interactive shell"))
 }
 
+#[maybe_async]
 pub fn handle_matches<C, D>(
     wallet: &Wallet<C, D>,
     matches: ArgMatches<'_>,
@@ -287,7 +288,7 @@ where
     if let Some(_sub_matches) = matches.subcommand_matches("get_new_address") {
         Ok(Some(format!("{}", wallet.get_new_address()?)))
     } else if let Some(_sub_matches) = matches.subcommand_matches("sync") {
-        wallet.sync(None, None)?;
+        maybe_await!(wallet.sync(None, None))?;
         Ok(None)
     } else if let Some(_sub_matches) = matches.subcommand_matches("list_unspent") {
         let mut res = String::new();
@@ -382,7 +383,7 @@ where
             panic!("Missing `psbt` and `tx` option");
         };
 
-        let txid = wallet.broadcast(tx)?;
+        let txid = maybe_await!(wallet.broadcast(tx))?;
 
         Ok(Some(format!("TXID: {}", txid)))
     } else if let Some(sub_matches) = matches.subcommand_matches("extract_psbt") {

src/lib.rs

@@ -5,6 +5,12 @@ extern crate serde;
 #[macro_use]
 extern crate serde_json;
 
+#[cfg(target_arch = "wasm32")]
+#[macro_use]
+extern crate async_trait;
+#[macro_use]
+extern crate magical_macros;
+
 #[cfg(test)]
 #[macro_use]
 extern crate lazy_static;

src/wallet/mod.rs

@@ -709,6 +709,7 @@ where
     B: OnlineBlockchain,
     D: BatchDatabase,
 {
+    #[maybe_async]
     pub fn new(
         descriptor: &str,
         change_descriptor: Option<&str>,
@@ -738,7 +739,7 @@ where
             None => None,
         };
 
-        let current_height = Some(client.get_height()? as u32);
+        let current_height = Some(maybe_await!(client.get_height())? as u32);
 
         Ok(Wallet {
             descriptor,
@@ -752,6 +753,7 @@ where
         })
     }
 
+    #[maybe_async]
     pub fn sync(
         &self,
         max_address: Option<u32>,
@@ -811,15 +813,16 @@ where
             self.database.borrow_mut().commit_batch(address_batch)?;
         }
 
-        self.client.borrow_mut().sync(
+        maybe_await!(self.client.borrow_mut().sync(
             None,
             self.database.borrow_mut().deref_mut(),
             noop_progress(),
-        )
+        ))
     }
 
+    #[maybe_async]
     pub fn broadcast(&self, tx: Transaction) -> Result<Txid, Error> {
-        self.client.borrow_mut().broadcast(&tx)?;
+        maybe_await!(self.client.borrow_mut().broadcast(&tx))?;
 
         Ok(tx.txid())
     }