machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #4608 from mempool/mononaut/goggle-index-bugfix

Browse Source 
Fix version update & error handling in Goggles indexing
This commit is contained in:
softsimon 2024-01-24 23:08:31 +07:00 committed by GitHub
commit 09ce1bd458
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 61 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

102
backend/src/api/blocks.ts
View File

@ -606,60 +606,66 @@ class Blocks {
logger.debug(`Classifying blocks and templates from #${currentBlockHeight} to #${minHeight}`, logger.tags.goggles);
for (let height = currentBlockHeight; height >= 0; height--) {
let txs: TransactionExtended[] | null = null;
if (unclassifiedBlocks[height]) {
const blockHash = unclassifiedBlocks[height];
// fetch transactions
txs = (await bitcoinApi.$getTxsForBlock(blockHash)).map(tx => transactionUtils.extendTransaction(tx));
// add CPFP
const cpfpSummary = Common.calculateCpfp(height, txs, true);
// classify
const { transactions: classifiedTxs } = this.summarizeBlockTransactions(blockHash, cpfpSummary.transactions);
BlocksSummariesRepository.$saveTransactions(height, blockHash, classifiedTxs, 1);
}
if (unclassifiedTemplates[height]) {
// classify template
const blockHash = unclassifiedTemplates[height];
const template = await BlocksSummariesRepository.$getTemplate(blockHash);
const alreadyClassified = template?.transactions.reduce((classified, tx) => (classified || tx.flags > 0), false);
let classifiedTemplate = template?.transactions || [];
if (!alreadyClassified) {
const templateTxs: (TransactionExtended | TransactionClassified)[] = [];
const blockTxMap: { [txid: string]: TransactionExtended } = {};
for (const tx of (txs || [])) {
blockTxMap[tx.txid] = tx;
}
for (const templateTx of (template?.transactions || [])) {
let tx: TransactionExtended | null = blockTxMap[templateTx.txid];
if (!tx) {
try {
tx = await transactionUtils.$getTransactionExtended(templateTx.txid, false, true, false);
} catch (e) {
// transaction probably not found
}
}
templateTxs.push(tx || templateTx);
}
const cpfpSummary = Common.calculateCpfp(height, txs?.filter(tx => tx.effectiveFeePerVsize != null) as TransactionExtended[], true);
try {
let txs: TransactionExtended[] | null = null;
if (unclassifiedBlocks[height]) {
const blockHash = unclassifiedBlocks[height];
// fetch transactions
txs = (await bitcoinApi.$getTxsForBlock(blockHash)).map(tx => transactionUtils.extendTransaction(tx)) || [];
// add CPFP
const cpfpSummary = Common.calculateCpfp(height, txs, true);
// classify
const { transactions: classifiedTxs } = this.summarizeBlockTransactions(blockHash, cpfpSummary.transactions);
const classifiedTxMap: { [txid: string]: TransactionClassified } = {};
for (const tx of classifiedTxs) {
classifiedTxMap[tx.txid] = tx;
}
classifiedTemplate = classifiedTemplate.map(tx => {
if (classifiedTxMap[tx.txid]) {
tx.flags = classifiedTxMap[tx.txid].flags || 0;
}
return tx;
});
await BlocksSummariesRepository.$saveTransactions(height, blockHash, classifiedTxs, 1);
}
BlocksSummariesRepository.$saveTemplate({ height, template: { id: blockHash, transactions: classifiedTemplate }, version: 1 });
if (unclassifiedTemplates[height]) {
// classify template
const blockHash = unclassifiedTemplates[height];
const template = await BlocksSummariesRepository.$getTemplate(blockHash);
const alreadyClassified = template?.transactions?.reduce((classified, tx) => (classified || tx.flags > 0), false);
let classifiedTemplate = template?.transactions || [];
if (!alreadyClassified) {
const templateTxs: (TransactionExtended | TransactionClassified)[] = [];
const blockTxMap: { [txid: string]: TransactionExtended } = {};
for (const tx of (txs || [])) {
blockTxMap[tx.txid] = tx;
}
for (const templateTx of (template?.transactions || [])) {
let tx: TransactionExtended | null = blockTxMap[templateTx.txid];
if (!tx) {
try {
tx = await transactionUtils.$getTransactionExtended(templateTx.txid, false, true, false);
} catch (e) {
// transaction probably not found
}
}
templateTxs.push(tx || templateTx);
}
const cpfpSummary = Common.calculateCpfp(height, templateTxs?.filter(tx => tx['effectiveFeePerVsize'] != null) as TransactionExtended[], true);
// classify
const { transactions: classifiedTxs } = this.summarizeBlockTransactions(blockHash, cpfpSummary.transactions);
const classifiedTxMap: { [txid: string]: TransactionClassified } = {};
for (const tx of classifiedTxs) {
classifiedTxMap[tx.txid] = tx;
}
classifiedTemplate = classifiedTemplate.map(tx => {
if (classifiedTxMap[tx.txid]) {
tx.flags = classifiedTxMap[tx.txid].flags || 0;
}
return tx;
});
}
await BlocksSummariesRepository.$saveTemplate({ height, template: { id: blockHash, transactions: classifiedTemplate }, version: 1 });
}
} catch (e) {
logger.warn(`Failed to classify template or block summary at ${height}`, logger.tags.goggles);
}
// timing & logging
indexedThisRun++;
indexedTotal++;
if (unclassifiedBlocks[height] || unclassifiedTemplates[height]) {
indexedThisRun++;
indexedTotal++;
}
const elapsedSeconds = (Date.now() - timer) / 1000;
if (elapsedSeconds > 5) {
const perSecond = indexedThisRun / elapsedSeconds;

1
backend/src/mempool.interfaces.ts
View File

@ -281,6 +281,7 @@ export interface BlockExtended extends IEsploraApi.Block {
export interface BlockSummary {
id: string;
transactions: TransactionClassified[];
version?: number;
}
export interface AuditSummary extends BlockAudit {

10
backend/src/repositories/BlocksSummariesRepository.ts
View File

@ -23,8 +23,8 @@ class BlocksSummariesRepository {
await DB.query(`
INSERT INTO blocks_summaries
SET height = ?, transactions = ?, id = ?, version = ?
ON DUPLICATE KEY UPDATE transactions = ?`,
[blockHeight, transactionsStr, blockId, version, transactionsStr]);
ON DUPLICATE KEY UPDATE transactions = ?, version = ?`,
[blockHeight, transactionsStr, blockId, version, transactionsStr, version]);
} catch (e: any) {
logger.debug(`Cannot save block summary transactions for ${blockId}. Reason: ${e instanceof Error ? e.message : e}`);
throw e;
@ -39,8 +39,9 @@ class BlocksSummariesRepository {
INSERT INTO blocks_templates (id, template, version)
VALUE (?, ?, ?)
ON DUPLICATE KEY UPDATE
template = ?
`, [blockId, transactions, params.version, transactions]);
template = ?,
version = ?
`, [blockId, transactions, params.version, transactions, params.version]);
} catch (e: any) {
if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
logger.debug(`Cannot save block template for ${blockId} because it has already been indexed, ignoring`);
@ -57,6 +58,7 @@ class BlocksSummariesRepository {
return {
id: templates[0].id,
transactions: JSON.parse(templates[0].template),
version: templates[0].version,
};
}
} catch (e) {