Merge branch 'master' into merged-expected-block-fees

backend/src/api/bitcoin/bitcoin-api.ts

@@ -415,12 +415,38 @@ class BitcoinApi implements AbstractBitcoinApi {
       vin.inner_witnessscript_asm = this.convertScriptSigAsm(witnessScript);
     }
 
-    if (vin.prevout.scriptpubkey_type === 'v1_p2tr' && vin.witness && vin.witness.length > 1) {
+    if (vin.prevout.scriptpubkey_type === 'v1_p2tr' && vin.witness) {
-      const witnessScript = vin.witness[vin.witness.length - 2];
+      const witnessScript = this.witnessToP2TRScript(vin.witness);
-      vin.inner_witnessscript_asm = this.convertScriptSigAsm(witnessScript);
+      if (witnessScript !== null) {
+        vin.inner_witnessscript_asm = this.convertScriptSigAsm(witnessScript);
+      }
     }
   }
 
+  /**
+   * This function must only be called when we know the witness we are parsing
+   * is a taproot witness.
+   * @param witness An array of hex strings that represents the witness stack of
+   *                the input.
+   * @returns null if the witness is not a script spend, and the hex string of
+   *          the script item if it is a script spend.
+   */
+  private witnessToP2TRScript(witness: string[]): string | null {
+    if (witness.length < 2) return null;
+    // Note: see BIP341 for parsing details of witness stack
+
+    // If there are at least two witness elements, and the first byte of the
+    // last element is 0x50, this last element is called annex a and
+    // is removed from the witness stack.
+    const hasAnnex = witness[witness.length - 1].substring(0, 2) === '50';
+    // If there are at least two witness elements left, script path spending is used.
+    // Call the second-to-last stack element s, the script.
+    // (Note: this phrasing from BIP341 assumes we've *removed* the annex from the stack)
+    if (hasAnnex && witness.length < 3) return null;
+    const positionOfScript = hasAnnex ? witness.length - 3 : witness.length - 2;
+    return witness[positionOfScript];
+  }
+
 }
 
 export default BitcoinApi;

backend/src/api/blocks.ts

@@ -602,6 +602,14 @@ class Blocks {
       const block = BitcoinApi.convertBlock(verboseBlock);
       const txIds: string[] = await bitcoinApi.$getTxIdsForBlock(blockHash);
       const transactions = await this.$getTransactionsExtended(blockHash, block.height, false, false, true);
+      if (config.MEMPOOL.BACKEND !== 'esplora') {
+        // fill in missing transaction fee data from verboseBlock
+        for (let i = 0; i < transactions.length; i++) {
+          if (!transactions[i].fee && transactions[i].txid === verboseBlock.tx[i].txid) {
+            transactions[i].fee = verboseBlock.tx[i].fee * 100_000_000;
+          }
+        }
+      }
       const cpfpSummary: CpfpSummary = Common.calculateCpfp(block.height, transactions);
       const blockExtended: BlockExtended = await this.$getBlockExtended(block, cpfpSummary.transactions);
       const blockSummary: BlockSummary = this.summarizeBlock(verboseBlock);

backend/src/api/database-migration.ts

@@ -523,9 +523,21 @@ class DatabaseMigration {
     }
 
     if (databaseSchemaVersion < 61 && isBitcoin === true) {
-      await this.$executeQuery('ALTER TABLE `blocks_audits` ADD expected_fees BIGINT UNSIGNED NOT NULL DEFAULT "0"');
+      // Break block templates into their own table
+      if (! await this.$checkIfTableExists('blocks_templates')) {
+        await this.$executeQuery('CREATE TABLE blocks_templates AS SELECT id, template FROM blocks_summaries WHERE template != "[]"');
+      }
+      await this.$executeQuery('ALTER TABLE blocks_templates MODIFY template JSON DEFAULT "[]"');
+      await this.$executeQuery('ALTER TABLE blocks_templates ADD PRIMARY KEY (id)');
+      await this.$executeQuery('ALTER TABLE blocks_summaries DROP COLUMN template');
       await this.updateToSchemaVersion(61);
     }
+
+    if (databaseSchemaVersion < 62 && isBitcoin === true) {
+      await this.$executeQuery('ALTER TABLE `blocks_audits` ADD expected_fees BIGINT UNSIGNED NOT NULL DEFAULT "0"');
+      await this.updateToSchemaVersion(62);
+    }
+
   }
 
   /**

backend/src/api/transaction-utils.ts

@@ -59,8 +59,8 @@ class TransactionUtils {
       feePerVsize: feePerVbytes,
       effectiveFeePerVsize: feePerVbytes,
     }, transaction);
-    if (!transaction?.status?.confirmed) {
+    if (!transaction?.status?.confirmed && !transactionExtended.firstSeen) {
-      transactionExtended.firstSeen = Math.round((new Date().getTime() / 1000));
+      transactionExtended.firstSeen = Math.round((Date.now() / 1000));
     }
     return transactionExtended;
   }
@@ -83,8 +83,8 @@ class TransactionUtils {
       adjustedFeePerVsize: adjustedFeePerVsize,
       effectiveFeePerVsize: adjustedFeePerVsize,
     });
-    if (!transaction?.status?.confirmed) {
+    if (!transactionExtended?.status?.confirmed && !transactionExtended.firstSeen) {
-      transactionExtended.firstSeen = Math.round((new Date().getTime() / 1000));
+      transactionExtended.firstSeen = Math.round((Date.now() / 1000));
     }
     return transactionExtended;
   }

backend/src/repositories/BlocksAuditsRepository.ts

@@ -14,7 +14,6 @@ class BlocksAuditRepositories {
         logger.debug(`Cannot save block audit for block ${audit.hash} because it has already been indexed, ignoring`);
       } else {
         logger.err(`Cannot save block audit into db. Reason: ` + (e instanceof Error ? e.message : e));
-        throw e;
       }
     }
   }
@@ -55,6 +54,7 @@ class BlocksAuditRepositories {
         transactions, template, missing_txs as missingTxs, added_txs as addedTxs, fresh_txs as freshTxs, sigop_txs as sigopTxs, match_rate as matchRate, expected_fees as expectedFees
         FROM blocks_audits
         JOIN blocks ON blocks.hash = blocks_audits.hash
+        JOIN blocks_templates ON blocks_templates.id = blocks_audits.hash
         JOIN blocks_summaries ON blocks_summaries.id = blocks_audits.hash
         WHERE blocks_audits.hash = "${hash}"
       `);

backend/src/repositories/BlocksSummariesRepository.ts

@@ -36,17 +36,16 @@ class BlocksSummariesRepository {
     try {
       const transactions = JSON.stringify(params.template?.transactions || []);
       await DB.query(`
-        INSERT INTO blocks_summaries (height, id, transactions, template)
+        INSERT INTO blocks_templates (id, template)
-        VALUE (?, ?, ?, ?)
+        VALUE (?, ?)
         ON DUPLICATE KEY UPDATE
           template = ?
-      `, [params.height, blockId, '[]', transactions, transactions]);
+      `, [blockId, transactions, transactions]);
     } catch (e: any) {
       if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
         logger.debug(`Cannot save block template for ${blockId} because it has already been indexed, ignoring`);
       } else {
-        logger.debug(`Cannot save block template for ${blockId}. Reason: ${e instanceof Error ? e.message : e}`);
+        logger.warn(`Cannot save block template for ${blockId}. Reason: ${e instanceof Error ? e.message : e}`);
-        throw e;
       }
     }
   }

frontend/src/app/components/statistics/statistics.component.ts

@@ -196,7 +196,7 @@ export class StatisticsComponent implements OnInit {
         this.feeLevelDropdownData.push({
           fee: fee,
           range,
-          color: _chartColors[i - 1],
+          color: _chartColors[i],
         });
       }
     });