From f95c16a78e83c25a7bb55b2ea07e49ca6cb10f06 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Sat, 13 May 2023 15:28:29 +0200 Subject: [PATCH 1/7] [audit] warn if we cannot save templates and remove exception re-throw --- backend/src/repositories/BlocksSummariesRepository.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/backend/src/repositories/BlocksSummariesRepository.ts b/backend/src/repositories/BlocksSummariesRepository.ts index f2560fbe7..3a883585a 100644 --- a/backend/src/repositories/BlocksSummariesRepository.ts +++ b/backend/src/repositories/BlocksSummariesRepository.ts @@ -45,8 +45,7 @@ class BlocksSummariesRepository { if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart logger.debug(`Cannot save block template for ${blockId} because it has already been indexed, ignoring`); } else { - logger.debug(`Cannot save block template for ${blockId}. Reason: ${e instanceof Error ? e.message : e}`); - throw e; + logger.warn(`Cannot save block template for ${blockId}. Reason: ${e instanceof Error ? e.message : e}`); } } } From b171ed6dd00a859506eee677cc4654c134a09709 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Thu, 25 May 2023 17:39:45 -0400 Subject: [PATCH 2/7] Break block templates into their own db table --- backend/src/api/database-migration.ts | 14 +++++++++++++- backend/src/repositories/BlocksAuditsRepository.ts | 1 + .../src/repositories/BlocksSummariesRepository.ts | 6 +++--- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index 21c87f9e2..e777d8adb 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository'; import { RowDataPacket } from 'mysql2'; class DatabaseMigration { - private static currentVersion = 60; + private static currentVersion = 61; private queryTimeout = 3600_000; private statisticsAddedIndexed = false; private uniqueLogs: string[] = []; @@ -521,6 +521,18 @@ class DatabaseMigration { await this.$executeQuery('ALTER TABLE `blocks_audits` ADD sigop_txs JSON DEFAULT "[]"'); await this.updateToSchemaVersion(60); } + + if (databaseSchemaVersion < 61 && isBitcoin === true) { + // Break block templates into their own table + if (! await this.$checkIfTableExists('blocks_templates')) { + await this.$executeQuery('CREATE TABLE blocks_templates AS SELECT id, template FROM blocks_summaries WHERE template != "[]"'); + } + await this.$executeQuery('ALTER TABLE blocks_templates MODIFY template JSON DEFAULT "[]"'); + await this.$executeQuery('ALTER TABLE blocks_templates ADD PRIMARY KEY (id)'); + await this.$executeQuery('ALTER TABLE blocks_summaries DROP COLUMN template'); + await this.updateToSchemaVersion(61); + } + } /** diff --git a/backend/src/repositories/BlocksAuditsRepository.ts b/backend/src/repositories/BlocksAuditsRepository.ts index 33075f43c..656340743 100644 --- a/backend/src/repositories/BlocksAuditsRepository.ts +++ b/backend/src/repositories/BlocksAuditsRepository.ts @@ -55,6 +55,7 @@ class BlocksAuditRepositories { transactions, template, missing_txs as missingTxs, added_txs as addedTxs, fresh_txs as freshTxs, sigop_txs as sigopTxs, match_rate as matchRate FROM blocks_audits JOIN blocks ON blocks.hash = blocks_audits.hash + JOIN blocks_templates ON blocks_templates.id = blocks_audits.hash JOIN blocks_summaries ON blocks_summaries.id = blocks_audits.hash WHERE blocks_audits.hash = "${hash}" `); diff --git a/backend/src/repositories/BlocksSummariesRepository.ts b/backend/src/repositories/BlocksSummariesRepository.ts index f2560fbe7..87d2617e6 100644 --- a/backend/src/repositories/BlocksSummariesRepository.ts +++ b/backend/src/repositories/BlocksSummariesRepository.ts @@ -36,11 +36,11 @@ class BlocksSummariesRepository { try { const transactions = JSON.stringify(params.template?.transactions || []); await DB.query(` - INSERT INTO blocks_summaries (height, id, transactions, template) - VALUE (?, ?, ?, ?) + INSERT INTO blocks_templates (id, template) + VALUE (?, ?) ON DUPLICATE KEY UPDATE template = ? - `, [params.height, blockId, '[]', transactions, transactions]); + `, [blockId, transactions, transactions]); } catch (e: any) { if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart logger.debug(`Cannot save block template for ${blockId} because it has already been indexed, ignoring`); From 37dd95a4a01094e0947261cc55026d81147563f5 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Sun, 4 Jun 2023 12:36:27 -0400 Subject: [PATCH 3/7] fix firstSeen reset bug --- backend/src/api/transaction-utils.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/src/api/transaction-utils.ts b/backend/src/api/transaction-utils.ts index 2d1f0f955..8523a938e 100644 --- a/backend/src/api/transaction-utils.ts +++ b/backend/src/api/transaction-utils.ts @@ -59,8 +59,8 @@ class TransactionUtils { feePerVsize: feePerVbytes, effectiveFeePerVsize: feePerVbytes, }, transaction); - if (!transaction?.status?.confirmed) { - transactionExtended.firstSeen = Math.round((new Date().getTime() / 1000)); + if (!transaction?.status?.confirmed && !transactionExtended.firstSeen) { + transactionExtended.firstSeen = Math.round((Date.now() / 1000)); } return transactionExtended; } @@ -83,8 +83,8 @@ class TransactionUtils { adjustedFeePerVsize: adjustedFeePerVsize, effectiveFeePerVsize: adjustedFeePerVsize, }); - if (!transaction?.status?.confirmed) { - transactionExtended.firstSeen = Math.round((new Date().getTime() / 1000)); + if (!transactionExtended?.status?.confirmed && !transactionExtended.firstSeen) { + transactionExtended.firstSeen = Math.round((Date.now() / 1000)); } return transactionExtended; } From 9e1de656c1d05a288a0d66a43f491b107ae63fdf Mon Sep 17 00:00:00 2001 From: junderw Date: Mon, 5 Jun 2023 07:21:55 -0700 Subject: [PATCH 4/7] Fix: Annex parsing for p2tr on bitcoind/romanz backends --- backend/src/api/bitcoin/bitcoin-api.ts | 32 +++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/backend/src/api/bitcoin/bitcoin-api.ts b/backend/src/api/bitcoin/bitcoin-api.ts index e20fe9e34..307736737 100644 --- a/backend/src/api/bitcoin/bitcoin-api.ts +++ b/backend/src/api/bitcoin/bitcoin-api.ts @@ -415,12 +415,38 @@ class BitcoinApi implements AbstractBitcoinApi { vin.inner_witnessscript_asm = this.convertScriptSigAsm(witnessScript); } - if (vin.prevout.scriptpubkey_type === 'v1_p2tr' && vin.witness && vin.witness.length > 1) { - const witnessScript = vin.witness[vin.witness.length - 2]; - vin.inner_witnessscript_asm = this.convertScriptSigAsm(witnessScript); + if (vin.prevout.scriptpubkey_type === 'v1_p2tr' && vin.witness) { + const witnessScript = this.witnessToP2TRScript(vin.witness); + if (witnessScript !== null) { + vin.inner_witnessscript_asm = this.convertScriptSigAsm(witnessScript); + } } } + /** + * This function must only be called when we know the witness we are parsing + * is a taproot witness. + * @param witness An array of hex strings that represents the witness stack of + * the input. + * @returns null if the witness is not a script spend, and the hex string of + * the script item if it is a script spend. + */ + private witnessToP2TRScript(witness: string[]): string | null { + if (witness.length < 2) return null; + // Note: see BIP341 for parsing details of witness stack + + // If there are at least two witness elements, and the first byte of the + // last element is 0x50, this last element is called annex a and + // is removed from the witness stack. + const hasAnnex = witness[witness.length - 1].substring(0, 2) === '50'; + // If there are at least two witness elements left, script path spending is used. + // Call the second-to-last stack element s, the script. + // (Note: this phrasing from BIP341 assumes we've *removed* the annex from the stack) + if (hasAnnex && witness.length < 3) return null; + const positionOfScript = hasAnnex ? witness.length - 3 : witness.length - 2; + return witness[positionOfScript]; + } + } export default BitcoinApi; From 689319437a24c1499b389b07321497d176c3c805 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Mon, 5 Jun 2023 14:23:37 -0400 Subject: [PATCH 5/7] fix graph filter dropdown colors --- frontend/src/app/components/statistics/statistics.component.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/app/components/statistics/statistics.component.ts b/frontend/src/app/components/statistics/statistics.component.ts index 5e638af31..35137bff1 100644 --- a/frontend/src/app/components/statistics/statistics.component.ts +++ b/frontend/src/app/components/statistics/statistics.component.ts @@ -196,7 +196,7 @@ export class StatisticsComponent implements OnInit { this.feeLevelDropdownData.push({ fee: fee, range, - color: _chartColors[i - 1], + color: _chartColors[i], }); } }); From 386037d1db872fecca10b00ed46ea4c29fbc3094 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Fri, 26 May 2023 19:12:12 -0400 Subject: [PATCH 6/7] Fix missing fees in $updateBlocks without esplora --- backend/src/api/blocks.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index fc12b5998..9e56db027 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -600,6 +600,14 @@ class Blocks { const block = BitcoinApi.convertBlock(verboseBlock); const txIds: string[] = await bitcoinApi.$getTxIdsForBlock(blockHash); const transactions = await this.$getTransactionsExtended(blockHash, block.height, false, false, true); + if (config.MEMPOOL.BACKEND !== 'esplora') { + // fill in missing transaction fee data from verboseBlock + for (let i = 0; i < transactions.length; i++) { + if (!transactions[i].fee && transactions[i].txid === verboseBlock.tx[i].txid) { + transactions[i].fee = verboseBlock.tx[i].fee * 100_000_000; + } + } + } const cpfpSummary: CpfpSummary = Common.calculateCpfp(block.height, transactions); const blockExtended: BlockExtended = await this.$getBlockExtended(block, cpfpSummary.transactions); const blockSummary: BlockSummary = this.summarizeBlock(verboseBlock); From fd30bff9c6e4f6b707d58e5dcea5039de8ed3126 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Wed, 7 Jun 2023 18:04:21 +0200 Subject: [PATCH 7/7] don't throw when BlocksAuditRepositories.$saveAudit fails --- backend/src/repositories/BlocksAuditsRepository.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/src/repositories/BlocksAuditsRepository.ts b/backend/src/repositories/BlocksAuditsRepository.ts index c6156334b..1149b8a93 100644 --- a/backend/src/repositories/BlocksAuditsRepository.ts +++ b/backend/src/repositories/BlocksAuditsRepository.ts @@ -14,7 +14,6 @@ class BlocksAuditRepositories { logger.debug(`Cannot save block audit for block ${audit.hash} because it has already been indexed, ignoring`); } else { logger.err(`Cannot save block audit into db. Reason: ` + (e instanceof Error ? e.message : e)); - throw e; } } }