Merge bitcoindevkit/bdk#605: Fix sqlite database set_utxo to insert or update utxos

35feb107ed5969720ce54a6aa76b7b2176f6c7c1 [CI] Fix cont_integration test-blockchains to run all tests (Steve Myers)
24719081511969299459d01566b0d669a7b51f7b Update CHANGELOG with warning about sqlite-db deleted wallet data (Steve Myers)
0b1a399f4e556a981bb992cc9b1d34318e260a7c Update sqlite schema with unique index for utxos, change insert_utxo to upsert (Steve Myers)
cea79872d717b395560a344cc4dd0e022c3bd9a9 Update database tests to verify set_utxo upserts (Steve Myers)

Pull request description:

  ### Description

  This PR fixes #591 by:
  1. Add sqlite `MIGRATIONS` statements to remove duplicate utxos and add unique utxos index on txid and vout.
  2. Do an upsert (if insert fails update) instead of an insert in `set_utxo()`.
  3. Update database::test::test_utxo to also verify `set_utxo()` doesn't insert duplicate utxos.

  ### Notes to the reviewers

  I verified the updated `test_utxo` fails as expected before my fix and passes after the fix. I tested the new migrations using the below `bdk-cli` command and a manually updated sqlite db with duplicate utxos.
  ```shell
  cargo run --no-default-features --features cli,sqlite-db,esplora-ureq -- wallet -w test1 --descriptor "wpkh(tpubEBr4i6yk5nf5DAaJpsi9N2pPYBeJ7fZ5Z9rmN4977iYLCGco1VyjB9tvvuvYtfZzjD5A8igzgw3HeWeeKFmanHYqksqZXYXGsw5zjnj7KM9/*)" sync
  ```

  ### Checklists

  #### All Submissions:

  * [x] I've signed all my commits
  * [x] I followed the [contribution guidelines](https://github.com/bitcoindevkit/bdk/blob/master/CONTRIBUTING.md)
  * [x] I ran `cargo fmt` and `cargo clippy` before committing

  #### New Features:

  * [ ] I've added tests for the new feature
  * [ ] I've added docs for the new feature
  * [ ] I've updated `CHANGELOG.md`

  #### Bugfixes:

  * [ ] This pull request breaks the existing API
  * [x] I've added tests to reproduce the issue which are now passing
  * [x] I'm linking the issue being fixed by this PR

ACKs for top commit:
  danielabrozzoni:
    utACK 35feb107ed5969720ce54a6aa76b7b2176f6c7c1 - Code looks good, but I didn't do any local test to see if the db gets wiped

Tree-SHA512: 753c7a0cfd0e803b5e12f39181d9a718791c4ce229d5072e6498db75a7008e94d447b3d0b4b0c205e7a8f127f60102e12bac2d271b8bad3a3038856bfd54e99c

.github/workflows/cont_integration.yml

@@ -90,12 +90,16 @@ jobs:
       matrix:
         blockchain:
           - name: electrum
+            testprefix: blockchain::electrum::test
             features: test-electrum,verify
           - name: rpc
+            testprefix: blockchain::rpc::test
             features: test-rpc
           - name: esplora
+            testprefix: esplora
             features: test-esplora,use-esplora-reqwest,verify
           - name: esplora
+            testprefix: esplora
             features: test-esplora,use-esplora-ureq,verify
     steps:
       - name: Checkout
@@ -114,7 +118,7 @@ jobs:
           toolchain: stable
           override: true
       - name: Test
-        run: cargo test --no-default-features --features ${{ matrix.blockchain.features }} ${{ matrix.blockchain.name }}::bdk_blockchain_tests
+        run: cargo test --no-default-features --features ${{ matrix.blockchain.features }} ${{ matrix.blockchain.testprefix }}::bdk_blockchain_tests
 
   check-wasm:
     name: Check WASM

CHANGELOG.md

@@ -10,8 +10,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - New MSRV set to `1.56`
 - Unpinned tokio to `1`
 - Add traits to reuse `Blockchain`s across multiple wallets (`BlockchainFactory` and `StatelessBlockchain`).
-- Upgrade to rust-bitcoin `0.28` 
-
+- Upgrade to rust-bitcoin `0.28`
+- If using the `sqlite-db` feature all cached wallet data is deleted due to a possible UTXO inconsistency, a wallet.sync will recreate it  
 
 ## [v0.18.0] - [v0.17.0]
 

src/database/mod.rs

@@ -323,7 +323,8 @@ pub mod test {
         };
 
         tree.set_utxo(&utxo).unwrap();
-
+        tree.set_utxo(&utxo).unwrap();
+        assert_eq!(tree.iter_utxos().unwrap().len(), 1);
         assert_eq!(tree.get_utxo(&outpoint).unwrap(), Some(utxo));
     }
 

src/database/sqlite.rs

@@ -41,6 +41,16 @@ static MIGRATIONS: &[&str] = &[
     "INSERT INTO transaction_details SELECT txid, timestamp, received, sent, fee, height FROM transaction_details_old;",
     "DROP TABLE transaction_details_old;",
     "ALTER TABLE utxos ADD COLUMN is_spent;",
+    // drop all data due to possible inconsistencies with duplicate utxos, re-sync required
+    "DELETE FROM checksums;",
+    "DELETE FROM last_derivation_indices;",
+    "DELETE FROM script_pubkeys;",
+    "DELETE FROM sync_time;",
+    "DELETE FROM transaction_details;",
+    "DELETE FROM transactions;",
+    "DELETE FROM utxos;",
+    "DROP INDEX idx_txid_vout;",
+    "CREATE UNIQUE INDEX idx_utxos_txid_vout ON utxos(txid, vout);"
 ];
 
 /// Sqlite database stored on filesystem
@@ -86,7 +96,7 @@ impl SqliteDatabase {
         script: &[u8],
         is_spent: bool,
     ) -> Result<i64, Error> {
-        let mut statement = self.connection.prepare_cached("INSERT INTO utxos (value, keychain, vout, txid, script, is_spent) VALUES (:value, :keychain, :vout, :txid, :script, :is_spent)")?;
+        let mut statement = self.connection.prepare_cached("INSERT INTO utxos (value, keychain, vout, txid, script, is_spent) VALUES (:value, :keychain, :vout, :txid, :script, :is_spent) ON CONFLICT(txid, vout) DO UPDATE SET value=:value, keychain=:keychain, script=:script, is_spent=:is_spent")?;
         statement.execute(named_params! {
             ":value": value,
             ":keychain": keychain,