From aa86885e6bc497844227381c7fecf9b9db7d9944 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Mon, 20 Jun 2022 16:35:10 +0200 Subject: [PATCH] Set `block/:hash/summary` expiration to 1 month - Support re-org for block summaries --- backend/src/api/blocks.ts | 32 ++++++++++++++----- backend/src/api/database-migration.ts | 5 +-- backend/src/indexer.ts | 8 ++++- backend/src/repositories/BlocksRepository.ts | 2 ++ .../repositories/BlocksSummariesRepository.ts | 13 ++++++++ backend/src/routes.ts | 2 +- frontend/src/app/services/api.service.ts | 2 +- 7 files changed, 51 insertions(+), 13 deletions(-) diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 33b8c93ba..571cc0f3b 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -254,16 +254,22 @@ class Blocks { try { // Get all indexed block hash const indexedBlocks = await blocksRepository.$getIndexedBlocks(); - const indexedBlockSummariesHashes = await BlocksSummariesRepository.$getIndexedSummariesId(); + const indexedBlockSummariesHashesArray = await BlocksSummariesRepository.$getIndexedSummariesId(); + + const indexedBlockSummariesHashes = {}; // Use a map for faster seek during the indexing loop + for (const hash of indexedBlockSummariesHashesArray) { + indexedBlockSummariesHashes[hash] = true; + } // Logging - let totalIndexed = indexedBlockSummariesHashes.length; + let newlyIndexed = 0; + let totalIndexed = indexedBlockSummariesHashesArray.length; let indexedThisRun = 0; let timer = new Date().getTime() / 1000; const startedAt = new Date().getTime() / 1000; for (const block of indexedBlocks) { - if (indexedBlockSummariesHashes.includes(block.hash)) { + if (indexedBlockSummariesHashes[block.hash] === true) { continue; } @@ -284,7 +290,9 @@ class Blocks { // Logging indexedThisRun++; totalIndexed++; + newlyIndexed++; } + logger.notice(`Blocks summaries indexing completed: indexed ${newlyIndexed} blocks`); } catch (e) { logger.err(`Blocks summaries indexing failed. Reason: ${(e instanceof Error ? e.message : e)}`); } @@ -293,10 +301,10 @@ class Blocks { /** * [INDEXING] Index all blocks metadata for the mining dashboard */ - public async $generateBlockDatabase() { + public async $generateBlockDatabase(): Promise { const blockchainInfo = await bitcoinClient.getBlockchainInfo(); if (blockchainInfo.blocks !== blockchainInfo.headers) { // Wait for node to sync - return; + return false; } try { @@ -364,13 +372,16 @@ class Blocks { } catch (e) { logger.err('Block indexing failed. Trying again later. Reason: ' + (e instanceof Error ? e.message : e)); loadingIndicators.setProgress('block-indexing', 100); - return; + return false; } const chainValid = await BlocksRepository.$validateChain(); if (!chainValid) { indexer.reindex(); + return false; } + + return true; } public async $updateBlocks() { @@ -435,9 +446,12 @@ class Blocks { // We assume there won't be a reorg with more than 10 block depth await BlocksRepository.$deleteBlocksFrom(lastBlock['height'] - 10); await HashratesRepository.$deleteLastEntries(); + await BlocksSummariesRepository.$deleteBlocksFrom(lastBlock['height'] - 10); for (let i = 10; i >= 0; --i) { - await this.$indexBlock(lastBlock['height'] - i); + const newBlock = await this.$indexBlock(lastBlock['height'] - i); + await this.$getStrippedBlockTransactions(newBlock.id, true, true); } + logger.info(`Re-indexed 10 blocks and summaries`); } await blocksRepository.$saveBlockInDatabase(blockExtended); @@ -530,7 +544,9 @@ class Blocks { return blockExtended; } - public async $getStrippedBlockTransactions(hash: string, skipMemoryCache: boolean = false, skipDBLookup: boolean = false): Promise { + public async $getStrippedBlockTransactions(hash: string, skipMemoryCache: boolean = false, + skipDBLookup: boolean = false): Promise + { if (skipMemoryCache === false) { // Check the memory cache const cachedSummary = this.getBlockSummaries().find((b) => b.id === hash); diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index 901381d79..2b7b6ddea 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -521,8 +521,9 @@ class DatabaseMigration { height int(10) unsigned NOT NULL, id varchar(65) NOT NULL, transactions JSON NOT NULL, - PRIMARY KEY (id) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`; + PRIMARY KEY (id), + INDEX (height) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`; } public async $truncateIndexedData(tables: string[]) { diff --git a/backend/src/indexer.ts b/backend/src/indexer.ts index d46991707..96cca9f7f 100644 --- a/backend/src/indexer.ts +++ b/backend/src/indexer.ts @@ -29,7 +29,13 @@ class Indexer { this.indexerRunning = true; try { - await blocks.$generateBlockDatabase(); + const chainValid = await blocks.$generateBlockDatabase(); + if (chainValid === false) { + // Chain of block hash was invalid, so we need to reindex. Stop here and continue at the next iteration + this.indexerRunning = false; + return; + } + await this.$resetHashratesIndexingState(); await mining.$generateNetworkHashrateHistory(); await mining.$generatePoolHashrateHistory(); diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index bd39a4494..01b7622f3 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -6,6 +6,7 @@ import { prepareBlock } from '../utils/blocks-utils'; import PoolsRepository from './PoolsRepository'; import HashratesRepository from './HashratesRepository'; import { escape } from 'mysql2'; +import BlocksSummariesRepository from './BlocksSummariesRepository'; class BlocksRepository { /** @@ -495,6 +496,7 @@ class BlocksRepository { if (blocks[idx].previous_block_hash !== blocks[idx - 1].hash) { logger.warn(`Chain divergence detected at block ${blocks[idx - 1].height}, re-indexing newer blocks and hashrates`); await this.$deleteBlocksFrom(blocks[idx - 1].height); + await BlocksSummariesRepository.$deleteBlocksFrom(blocks[idx - 1].height); await HashratesRepository.$deleteHashratesFromTimestamp(blocks[idx - 1].timestamp - 604800); return false; } diff --git a/backend/src/repositories/BlocksSummariesRepository.ts b/backend/src/repositories/BlocksSummariesRepository.ts index 73b43ed6f..66c6b97f2 100644 --- a/backend/src/repositories/BlocksSummariesRepository.ts +++ b/backend/src/repositories/BlocksSummariesRepository.ts @@ -40,6 +40,19 @@ class BlocksSummariesRepository { return []; } + + /** + * Delete blocks from the database from blockHeight + */ + public async $deleteBlocksFrom(blockHeight: number) { + logger.info(`Delete newer blocks summary from height ${blockHeight} from the database`); + + try { + await DB.query(`DELETE FROM blocks_summaries where height >= ${blockHeight}`); + } catch (e) { + logger.err('Cannot delete indexed blocks summaries. Reason: ' + (e instanceof Error ? e.message : e)); + } + } } export default new BlocksSummariesRepository(); diff --git a/backend/src/routes.ts b/backend/src/routes.ts index 99f54a9f8..e63549d09 100644 --- a/backend/src/routes.ts +++ b/backend/src/routes.ts @@ -729,7 +729,7 @@ class Routes { public async getStrippedBlockTransactions(req: Request, res: Response) { try { const transactions = await blocks.$getStrippedBlockTransactions(req.params.hash); - res.setHeader('Expires', new Date(Date.now() + 1000 * 600).toUTCString()); + res.setHeader('Expires', new Date(Date.now() + 1000 * 3600 * 24 * 30).toUTCString()); res.json(transactions); } catch (e) { res.status(500).send(e instanceof Error ? e.message : e); diff --git a/frontend/src/app/services/api.service.ts b/frontend/src/app/services/api.service.ts index 8202fbb49..42c942dad 100644 --- a/frontend/src/app/services/api.service.ts +++ b/frontend/src/app/services/api.service.ts @@ -1,7 +1,7 @@ import { Injectable } from '@angular/core'; import { HttpClient, HttpParams } from '@angular/common/http'; import { CpfpInfo, OptimizedMempoolStats, AddressInformation, LiquidPegs, ITranslators, - PoolsStats, PoolStat, BlockExtended, TransactionStripped, RewardStats } from '../interfaces/node-api.interface'; + PoolStat, BlockExtended, TransactionStripped, RewardStats } from '../interfaces/node-api.interface'; import { Observable } from 'rxjs'; import { StateService } from './state.service'; import { WebsocketResponse } from '../interfaces/websocket.interface';