Merge pull request #3442 from mempool/nymkappa/reorg-keep-templates

When a re-org happens, keep the block templates for audit
This commit is contained in:
softsimon 2023-04-03 12:24:05 +09:00 committed by GitHub
commit 2ef340712f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 24 additions and 48 deletions

View File

@ -590,11 +590,10 @@ class Blocks {
if (!fastForwarded) { if (!fastForwarded) {
const lastBlock = await blocksRepository.$getBlockByHeight(blockExtended.height - 1); const lastBlock = await blocksRepository.$getBlockByHeight(blockExtended.height - 1);
if (lastBlock !== null && blockExtended.previousblockhash !== lastBlock.id) { if (lastBlock !== null && blockExtended.previousblockhash !== lastBlock.id) {
logger.warn(`Chain divergence detected at block ${lastBlock.height}, re-indexing most recent data`); logger.warn(`Chain divergence detected at block ${lastBlock.height}, re-indexing most recent data`, logger.tags.mining);
// We assume there won't be a reorg with more than 10 block depth // We assume there won't be a reorg with more than 10 block depth
await BlocksRepository.$deleteBlocksFrom(lastBlock.height - 10); await BlocksRepository.$deleteBlocksFrom(lastBlock.height - 10);
await HashratesRepository.$deleteLastEntries(); await HashratesRepository.$deleteLastEntries();
await BlocksSummariesRepository.$deleteBlocksFrom(lastBlock.height - 10);
await cpfpRepository.$deleteClustersFrom(lastBlock.height - 10); await cpfpRepository.$deleteClustersFrom(lastBlock.height - 10);
for (let i = 10; i >= 0; --i) { for (let i = 10; i >= 0; --i) {
const newBlock = await this.$indexBlock(lastBlock.height - i); const newBlock = await this.$indexBlock(lastBlock.height - i);
@ -605,7 +604,7 @@ class Blocks {
} }
await mining.$indexDifficultyAdjustments(); await mining.$indexDifficultyAdjustments();
await DifficultyAdjustmentsRepository.$deleteLastAdjustment(); await DifficultyAdjustmentsRepository.$deleteLastAdjustment();
logger.info(`Re-indexed 10 blocks and summaries. Also re-indexed the last difficulty adjustments. Will re-index latest hashrates in a few seconds.`); logger.info(`Re-indexed 10 blocks and summaries. Also re-indexed the last difficulty adjustments. Will re-index latest hashrates in a few seconds.`, logger.tags.mining);
indexer.reindex(); indexer.reindex();
} }
await blocksRepository.$saveBlockInDatabase(blockExtended); await blocksRepository.$saveBlockInDatabase(blockExtended);
@ -737,7 +736,7 @@ class Blocks {
// Index the response if needed // Index the response if needed
if (Common.blocksSummariesIndexingEnabled() === true) { if (Common.blocksSummariesIndexingEnabled() === true) {
await BlocksSummariesRepository.$saveSummary({height: block.height, mined: summary}); await BlocksSummariesRepository.$saveTransactions(block.height, block.hash, summary.transactions);
} }
return summary.transactions; return summary.transactions;
@ -853,7 +852,7 @@ class Blocks {
if (cleanBlock.fee_amt_percentiles === null) { if (cleanBlock.fee_amt_percentiles === null) {
const block = await bitcoinClient.getBlock(cleanBlock.hash, 2); const block = await bitcoinClient.getBlock(cleanBlock.hash, 2);
const summary = this.summarizeBlock(block); const summary = this.summarizeBlock(block);
await BlocksSummariesRepository.$saveSummary({ height: block.height, mined: summary }); await BlocksSummariesRepository.$saveTransactions(cleanBlock.height, cleanBlock.hash, summary.transactions);
cleanBlock.fee_amt_percentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(cleanBlock.hash); cleanBlock.fee_amt_percentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(cleanBlock.hash);
} }
if (cleanBlock.fee_amt_percentiles !== null) { if (cleanBlock.fee_amt_percentiles !== null) {

View File

@ -466,30 +466,6 @@ class BlocksRepository {
} }
} }
/**
* Get one block by hash
*/
public async $getBlockByHash(hash: string): Promise<object | null> {
try {
const query = `
SELECT ${BLOCK_DB_FIELDS}
FROM blocks
JOIN pools ON blocks.pool_id = pools.id
WHERE hash = ?;
`;
const [rows]: any[] = await DB.query(query, [hash]);
if (rows.length <= 0) {
return null;
}
return await this.formatDbBlockIntoExtendedBlock(rows[0]);
} catch (e) {
logger.err(`Cannot get indexed block ${hash}. Reason: ` + (e instanceof Error ? e.message : e));
throw e;
}
}
/** /**
* Return blocks difficulty * Return blocks difficulty
*/ */
@ -599,7 +575,6 @@ class BlocksRepository {
if (blocks[idx].previous_block_hash !== blocks[idx - 1].hash) { if (blocks[idx].previous_block_hash !== blocks[idx - 1].hash) {
logger.warn(`Chain divergence detected at block ${blocks[idx - 1].height}`); logger.warn(`Chain divergence detected at block ${blocks[idx - 1].height}`);
await this.$deleteBlocksFrom(blocks[idx - 1].height); await this.$deleteBlocksFrom(blocks[idx - 1].height);
await BlocksSummariesRepository.$deleteBlocksFrom(blocks[idx - 1].height);
await HashratesRepository.$deleteHashratesFromTimestamp(blocks[idx - 1].timestamp - 604800); await HashratesRepository.$deleteHashratesFromTimestamp(blocks[idx - 1].timestamp - 604800);
await DifficultyAdjustmentsRepository.$deleteAdjustementsFromHeight(blocks[idx - 1].height); await DifficultyAdjustmentsRepository.$deleteAdjustementsFromHeight(blocks[idx - 1].height);
return false; return false;
@ -619,7 +594,7 @@ class BlocksRepository {
* Delete blocks from the database from blockHeight * Delete blocks from the database from blockHeight
*/ */
public async $deleteBlocksFrom(blockHeight: number) { public async $deleteBlocksFrom(blockHeight: number) {
logger.info(`Delete newer blocks from height ${blockHeight} from the database`); logger.info(`Delete newer blocks from height ${blockHeight} from the database`, logger.tags.mining);
try { try {
await DB.query(`DELETE FROM blocks where height >= ${blockHeight}`); await DB.query(`DELETE FROM blocks where height >= ${blockHeight}`);
@ -997,6 +972,7 @@ class BlocksRepository {
} }
// If we're missing block summary related field, check if we can populate them on the fly now // If we're missing block summary related field, check if we can populate them on the fly now
// This is for example triggered upon re-org
if (Common.blocksSummariesIndexingEnabled() && if (Common.blocksSummariesIndexingEnabled() &&
(extras.medianFeeAmt === null || extras.feePercentiles === null)) (extras.medianFeeAmt === null || extras.feePercentiles === null))
{ {
@ -1004,7 +980,7 @@ class BlocksRepository {
if (extras.feePercentiles === null) { if (extras.feePercentiles === null) {
const block = await bitcoinClient.getBlock(dbBlk.id, 2); const block = await bitcoinClient.getBlock(dbBlk.id, 2);
const summary = blocks.summarizeBlock(block); const summary = blocks.summarizeBlock(block);
await BlocksSummariesRepository.$saveSummary({ height: block.height, mined: summary }); await BlocksSummariesRepository.$saveTransactions(dbBlk.height, dbBlk.hash, summary.transactions);
extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(dbBlk.id); extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(dbBlk.id);
} }
if (extras.feePercentiles !== null) { if (extras.feePercentiles !== null) {

View File

@ -1,6 +1,6 @@
import DB from '../database'; import DB from '../database';
import logger from '../logger'; import logger from '../logger';
import { BlockSummary } from '../mempool.interfaces'; import { BlockSummary, TransactionStripped } from '../mempool.interfaces';
class BlocksSummariesRepository { class BlocksSummariesRepository {
public async $getByBlockId(id: string): Promise<BlockSummary | undefined> { public async $getByBlockId(id: string): Promise<BlockSummary | undefined> {
@ -17,7 +17,7 @@ class BlocksSummariesRepository {
return undefined; return undefined;
} }
public async $saveSummary(params: { height: number, mined?: BlockSummary}) { public async $saveSummary(params: { height: number, mined?: BlockSummary}): Promise<void> {
const blockId = params.mined?.id; const blockId = params.mined?.id;
try { try {
const transactions = JSON.stringify(params.mined?.transactions || []); const transactions = JSON.stringify(params.mined?.transactions || []);
@ -37,6 +37,20 @@ class BlocksSummariesRepository {
} }
} }
public async $saveTransactions(blockHeight: number, blockId: string, transactions: TransactionStripped[]): Promise<void> {
try {
const transactionsStr = JSON.stringify(transactions);
await DB.query(`
INSERT INTO blocks_summaries
SET height = ?, transactions = ?, id = ?
ON DUPLICATE KEY UPDATE transactions = ?`,
[blockHeight, transactionsStr, blockId, transactionsStr]);
} catch (e: any) {
logger.debug(`Cannot save block summary transactions for ${blockId}. Reason: ${e instanceof Error ? e.message : e}`);
throw e;
}
}
public async $saveTemplate(params: { height: number, template: BlockSummary}) { public async $saveTemplate(params: { height: number, template: BlockSummary}) {
const blockId = params.template?.id; const blockId = params.template?.id;
try { try {
@ -68,19 +82,6 @@ class BlocksSummariesRepository {
return []; return [];
} }
/**
* Delete blocks from the database from blockHeight
*/
public async $deleteBlocksFrom(blockHeight: number) {
logger.info(`Delete newer blocks summary from height ${blockHeight} from the database`);
try {
await DB.query(`DELETE FROM blocks_summaries where height >= ${blockHeight}`);
} catch (e) {
logger.err('Cannot delete indexed blocks summaries. Reason: ' + (e instanceof Error ? e.message : e));
}
}
/** /**
* Get the fee percentiles if the block has already been indexed, [] otherwise * Get the fee percentiles if the block has already been indexed, [] otherwise
* *

View File

@ -220,7 +220,7 @@ class HashratesRepository {
* Delete hashrates from the database from timestamp * Delete hashrates from the database from timestamp
*/ */
public async $deleteHashratesFromTimestamp(timestamp: number) { public async $deleteHashratesFromTimestamp(timestamp: number) {
logger.info(`Delete newer hashrates from timestamp ${new Date(timestamp * 1000).toUTCString()} from the database`); logger.info(`Delete newer hashrates from timestamp ${new Date(timestamp * 1000).toUTCString()} from the database`, logger.tags.mining);
try { try {
await DB.query(`DELETE FROM hashrates WHERE hashrate_timestamp >= FROM_UNIXTIME(?)`, [timestamp]); await DB.query(`DELETE FROM hashrates WHERE hashrate_timestamp >= FROM_UNIXTIME(?)`, [timestamp]);