From 8ca2b2b5c03520ea0f7901a2ab8a3c6a4544c903 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 23 Jan 2024 17:18:47 +0000 Subject: [PATCH 1/2] Fix version update & error handling in Goggles indexing --- backend/src/api/blocks.ts | 96 ++++++++++--------- backend/src/mempool.interfaces.ts | 1 + .../repositories/BlocksSummariesRepository.ts | 10 +- 3 files changed, 57 insertions(+), 50 deletions(-) diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index b95756ae5..348c44b78 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -606,55 +606,59 @@ class Blocks { logger.debug(`Classifying blocks and templates from #${currentBlockHeight} to #${minHeight}`, logger.tags.goggles); for (let height = currentBlockHeight; height >= 0; height--) { - let txs: TransactionExtended[] | null = null; - if (unclassifiedBlocks[height]) { - const blockHash = unclassifiedBlocks[height]; - // fetch transactions - txs = (await bitcoinApi.$getTxsForBlock(blockHash)).map(tx => transactionUtils.extendTransaction(tx)); - // add CPFP - const cpfpSummary = Common.calculateCpfp(height, txs, true); - // classify - const { transactions: classifiedTxs } = this.summarizeBlockTransactions(blockHash, cpfpSummary.transactions); - BlocksSummariesRepository.$saveTransactions(height, blockHash, classifiedTxs, 1); - } - if (unclassifiedTemplates[height]) { - // classify template - const blockHash = unclassifiedTemplates[height]; - const template = await BlocksSummariesRepository.$getTemplate(blockHash); - const alreadyClassified = template?.transactions.reduce((classified, tx) => (classified || tx.flags > 0), false); - let classifiedTemplate = template?.transactions || []; - if (!alreadyClassified) { - const templateTxs: (TransactionExtended | TransactionClassified)[] = []; - const blockTxMap: { [txid: string]: TransactionExtended } = {}; - for (const tx of (txs || [])) { - blockTxMap[tx.txid] = tx; - } - for (const templateTx of (template?.transactions || [])) { - let tx: TransactionExtended | null = blockTxMap[templateTx.txid]; - if (!tx) { - try { - tx = await transactionUtils.$getTransactionExtended(templateTx.txid, false, true, false); - } catch (e) { - // transaction probably not found - } - } - templateTxs.push(tx || templateTx); - } - const cpfpSummary = Common.calculateCpfp(height, txs?.filter(tx => tx.effectiveFeePerVsize != null) as TransactionExtended[], true); + try { + let txs: TransactionExtended[] | null = null; + if (unclassifiedBlocks[height]) { + const blockHash = unclassifiedBlocks[height]; + // fetch transactions + txs = (await bitcoinApi.$getTxsForBlock(blockHash)).map(tx => transactionUtils.extendTransaction(tx)); + // add CPFP + const cpfpSummary = Common.calculateCpfp(height, txs, true); // classify const { transactions: classifiedTxs } = this.summarizeBlockTransactions(blockHash, cpfpSummary.transactions); - const classifiedTxMap: { [txid: string]: TransactionClassified } = {}; - for (const tx of classifiedTxs) { - classifiedTxMap[tx.txid] = tx; - } - classifiedTemplate = classifiedTemplate.map(tx => { - if (classifiedTxMap[tx.txid]) { - tx.flags = classifiedTxMap[tx.txid].flags || 0; - } - return tx; - }); + await BlocksSummariesRepository.$saveTransactions(height, blockHash, classifiedTxs, 1); } - BlocksSummariesRepository.$saveTemplate({ height, template: { id: blockHash, transactions: classifiedTemplate }, version: 1 }); + if (unclassifiedTemplates[height]) { + // classify template + const blockHash = unclassifiedTemplates[height]; + const template = await BlocksSummariesRepository.$getTemplate(blockHash); + const alreadyClassified = template?.transactions.reduce((classified, tx) => (classified || tx.flags > 0), false); + let classifiedTemplate = template?.transactions || []; + if (!alreadyClassified) { + const templateTxs: (TransactionExtended | TransactionClassified)[] = []; + const blockTxMap: { [txid: string]: TransactionExtended } = {}; + for (const tx of (txs || [])) { + blockTxMap[tx.txid] = tx; + } + for (const templateTx of (template?.transactions || [])) { + let tx: TransactionExtended | null = blockTxMap[templateTx.txid]; + if (!tx) { + try { + tx = await transactionUtils.$getTransactionExtended(templateTx.txid, false, true, false); + } catch (e) { + // transaction probably not found + } + } + templateTxs.push(tx || templateTx); + } + const cpfpSummary = Common.calculateCpfp(height, txs?.filter(tx => tx.effectiveFeePerVsize != null) as TransactionExtended[], true); + // classify + const { transactions: classifiedTxs } = this.summarizeBlockTransactions(blockHash, cpfpSummary.transactions); + const classifiedTxMap: { [txid: string]: TransactionClassified } = {}; + for (const tx of classifiedTxs) { + classifiedTxMap[tx.txid] = tx; + } + classifiedTemplate = classifiedTemplate.map(tx => { + if (classifiedTxMap[tx.txid]) { + tx.flags = classifiedTxMap[tx.txid].flags || 0; + } + return tx; + }); + } + await BlocksSummariesRepository.$saveTemplate({ height, template: { id: blockHash, transactions: classifiedTemplate }, version: 1 }); + } + } catch (e) { + logger.warn(`Failed to classify template or block summary at ${height}`, logger.tags.goggles); } // timing & logging diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index 09d251050..ead0a84ad 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -281,6 +281,7 @@ export interface BlockExtended extends IEsploraApi.Block { export interface BlockSummary { id: string; transactions: TransactionClassified[]; + version?: number; } export interface AuditSummary extends BlockAudit { diff --git a/backend/src/repositories/BlocksSummariesRepository.ts b/backend/src/repositories/BlocksSummariesRepository.ts index b440960e0..f85914e31 100644 --- a/backend/src/repositories/BlocksSummariesRepository.ts +++ b/backend/src/repositories/BlocksSummariesRepository.ts @@ -23,8 +23,8 @@ class BlocksSummariesRepository { await DB.query(` INSERT INTO blocks_summaries SET height = ?, transactions = ?, id = ?, version = ? - ON DUPLICATE KEY UPDATE transactions = ?`, - [blockHeight, transactionsStr, blockId, version, transactionsStr]); + ON DUPLICATE KEY UPDATE transactions = ?, version = ?`, + [blockHeight, transactionsStr, blockId, version, transactionsStr, version]); } catch (e: any) { logger.debug(`Cannot save block summary transactions for ${blockId}. Reason: ${e instanceof Error ? e.message : e}`); throw e; @@ -39,8 +39,9 @@ class BlocksSummariesRepository { INSERT INTO blocks_templates (id, template, version) VALUE (?, ?, ?) ON DUPLICATE KEY UPDATE - template = ? - `, [blockId, transactions, params.version, transactions]); + template = ?, + version = ? + `, [blockId, transactions, params.version, transactions, params.version]); } catch (e: any) { if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart logger.debug(`Cannot save block template for ${blockId} because it has already been indexed, ignoring`); @@ -57,6 +58,7 @@ class BlocksSummariesRepository { return { id: templates[0].id, transactions: JSON.parse(templates[0].template), + version: templates[0].version, }; } } catch (e) { From 09a727c00d678438d6e8f5747f274d92189c1bb9 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Wed, 24 Jan 2024 12:18:42 +0000 Subject: [PATCH 2/2] Fix more Goggles indexing bugs --- backend/src/api/blocks.ts | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 348c44b78..21818dc62 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -611,7 +611,7 @@ class Blocks { if (unclassifiedBlocks[height]) { const blockHash = unclassifiedBlocks[height]; // fetch transactions - txs = (await bitcoinApi.$getTxsForBlock(blockHash)).map(tx => transactionUtils.extendTransaction(tx)); + txs = (await bitcoinApi.$getTxsForBlock(blockHash)).map(tx => transactionUtils.extendTransaction(tx)) || []; // add CPFP const cpfpSummary = Common.calculateCpfp(height, txs, true); // classify @@ -622,7 +622,7 @@ class Blocks { // classify template const blockHash = unclassifiedTemplates[height]; const template = await BlocksSummariesRepository.$getTemplate(blockHash); - const alreadyClassified = template?.transactions.reduce((classified, tx) => (classified || tx.flags > 0), false); + const alreadyClassified = template?.transactions?.reduce((classified, tx) => (classified || tx.flags > 0), false); let classifiedTemplate = template?.transactions || []; if (!alreadyClassified) { const templateTxs: (TransactionExtended | TransactionClassified)[] = []; @@ -641,7 +641,7 @@ class Blocks { } templateTxs.push(tx || templateTx); } - const cpfpSummary = Common.calculateCpfp(height, txs?.filter(tx => tx.effectiveFeePerVsize != null) as TransactionExtended[], true); + const cpfpSummary = Common.calculateCpfp(height, templateTxs?.filter(tx => tx['effectiveFeePerVsize'] != null) as TransactionExtended[], true); // classify const { transactions: classifiedTxs } = this.summarizeBlockTransactions(blockHash, cpfpSummary.transactions); const classifiedTxMap: { [txid: string]: TransactionClassified } = {}; @@ -662,8 +662,10 @@ class Blocks { } // timing & logging - indexedThisRun++; - indexedTotal++; + if (unclassifiedBlocks[height] || unclassifiedTemplates[height]) { + indexedThisRun++; + indexedTotal++; + } const elapsedSeconds = (Date.now() - timer) / 1000; if (elapsedSeconds > 5) { const perSecond = indexedThisRun / elapsedSeconds;