From 72a603ac370c7b23121abb581041a8a6fde3c134 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Sat, 18 Jun 2022 16:48:02 +0200 Subject: [PATCH] Index block summaries in db --- backend/mempool-config.sample.json | 1 + backend/src/api/blocks.ts | 83 +++++++++++++++++-- backend/src/api/common.ts | 7 ++ backend/src/api/database-migration.ts | 15 +++- backend/src/config.ts | 2 + backend/src/indexer.ts | 1 + backend/src/repositories/BlocksRepository.ts | 13 +++ .../repositories/BlocksSummariesRepository.ts | 46 ++++++++++ docker/backend/mempool-config.json | 3 +- docker/backend/start.sh | 2 + production/mempool-config.mainnet.json | 1 + 11 files changed, 166 insertions(+), 8 deletions(-) create mode 100644 backend/src/repositories/BlocksSummariesRepository.ts diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index 77b571136..eedbf3e4c 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -13,6 +13,7 @@ "INITIAL_BLOCKS_AMOUNT": 8, "MEMPOOL_BLOCKS_AMOUNT": 8, "INDEXING_BLOCKS_AMOUNT": 11000, + "BLOCKS_SUMMARIES_INDEXING": false, "PRICE_FEED_UPDATE_INTERVAL": 600, "USE_SECOND_NODE_FOR_MINFEE": false, "EXTERNAL_ASSETS": [], diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index c013bbdd3..33b8c93ba 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -20,6 +20,7 @@ import indexer from '../indexer'; import fiatConversion from './fiat-conversion'; import RatesRepository from '../repositories/RatesRepository'; import poolsParser from './pools-parser'; +import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository'; class Blocks { private blocks: BlockExtended[] = []; @@ -242,6 +243,53 @@ class Blocks { } } + /** + * [INDEXING] Index all blocks summaries for the block txs visualization + */ + public async $generateBlocksSummariesDatabase() { + if (Common.blocksSummariesIndexingEnabled() === false) { + return; + } + + try { + // Get all indexed block hash + const indexedBlocks = await blocksRepository.$getIndexedBlocks(); + const indexedBlockSummariesHashes = await BlocksSummariesRepository.$getIndexedSummariesId(); + + // Logging + let totalIndexed = indexedBlockSummariesHashes.length; + let indexedThisRun = 0; + let timer = new Date().getTime() / 1000; + const startedAt = new Date().getTime() / 1000; + + for (const block of indexedBlocks) { + if (indexedBlockSummariesHashes.includes(block.hash)) { + continue; + } + + // Logging + const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - timer)); + if (elapsedSeconds > 5) { + const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt)); + const blockPerSeconds = Math.max(1, indexedThisRun / elapsedSeconds); + const progress = Math.round(totalIndexed / indexedBlocks.length * 10000) / 100; + const timeLeft = Math.round((indexedBlocks.length - totalIndexed) / blockPerSeconds); + logger.debug(`Indexing block summary for #${block.height} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${totalIndexed}/${indexedBlocks.length} (${progress}%) | elapsed: ${runningFor} seconds | left: ~${timeLeft} seconds`); + timer = new Date().getTime() / 1000; + indexedThisRun = 0; + } + + await this.$getStrippedBlockTransactions(block.hash, true, true); // This will index the block summary + + // Logging + indexedThisRun++; + totalIndexed++; + } + } catch (e) { + logger.err(`Blocks summaries indexing failed. Reason: ${(e instanceof Error ? e.message : e)}`); + } + } + /** * [INDEXING] Index all blocks metadata for the mining dashboard */ @@ -292,7 +340,7 @@ class Blocks { const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - timer)); if (elapsedSeconds > 5 || blockHeight === lastBlockToIndex) { const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt)); - const blockPerSeconds = Math.max(1, Math.round(indexedThisRun / elapsedSeconds)); + const blockPerSeconds = Math.max(1, indexedThisRun / elapsedSeconds); const progress = Math.round(totalIndexed / indexingBlockAmount * 10000) / 100; const timeLeft = Math.round((indexingBlockAmount - totalIndexed) / blockPerSeconds); logger.debug(`Indexing block #${blockHeight} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${totalIndexed}/${indexingBlockAmount} (${progress}%) | elapsed: ${runningFor} seconds | left: ~${timeLeft} seconds`); @@ -392,6 +440,11 @@ class Blocks { } } await blocksRepository.$saveBlockInDatabase(blockExtended); + + // Save blocks summary for visualization if it's enabled + if (Common.blocksSummariesIndexingEnabled() === true) { + await this.$getStrippedBlockTransactions(blockExtended.id, true); + } } } if (fiatConversion.ratesInitialized === true && config.DATABASE.ENABLED === true) { @@ -477,14 +530,32 @@ class Blocks { return blockExtended; } - public async $getStrippedBlockTransactions(hash: string): Promise { - // Check the memory cache - const cachedSummary = this.getBlockSummaries().find((b) => b.id === hash); - if (cachedSummary) { - return cachedSummary.transactions; + public async $getStrippedBlockTransactions(hash: string, skipMemoryCache: boolean = false, skipDBLookup: boolean = false): Promise { + if (skipMemoryCache === false) { + // Check the memory cache + const cachedSummary = this.getBlockSummaries().find((b) => b.id === hash); + if (cachedSummary) { + return cachedSummary.transactions; + } } + + // Check if it's indexed in db + if (skipDBLookup === false && Common.blocksSummariesIndexingEnabled() === true) { + const indexedSummary = await BlocksSummariesRepository.$getByBlockId(hash); + if (indexedSummary !== undefined) { + return indexedSummary.transactions; + } + } + + // Call Core RPC const block = await bitcoinClient.getBlock(hash, 2); const summary = this.summarizeBlock(block); + + // Index the response if needed + if (Common.blocksSummariesIndexingEnabled() === true) { + await BlocksSummariesRepository.$saveSummary(block.height, summary); + } + return summary.transactions; } diff --git a/backend/src/api/common.ts b/backend/src/api/common.ts index d4b57f204..d1c8cecbb 100644 --- a/backend/src/api/common.ts +++ b/backend/src/api/common.ts @@ -177,4 +177,11 @@ export class Common { config.MEMPOOL.INDEXING_BLOCKS_AMOUNT !== 0 ); } + + static blocksSummariesIndexingEnabled(): boolean { + return ( + Common.indexingEnabled() && + config.MEMPOOL.BLOCKS_SUMMARIES_INDEXING === true + ); + } } diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index 5ac63740b..901381d79 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -4,7 +4,7 @@ import logger from '../logger'; import { Common } from './common'; class DatabaseMigration { - private static currentVersion = 19; + private static currentVersion = 20; private queryTimeout = 120000; private statisticsAddedIndexed = false; private uniqueLogs: string[] = []; @@ -217,6 +217,10 @@ class DatabaseMigration { if (databaseSchemaVersion < 19) { await this.$executeQuery(this.getCreateRatesTableQuery(), await this.$checkIfTableExists('rates')); } + + if (databaseSchemaVersion < 20 && isBitcoin === true) { + await this.$executeQuery(this.getCreateBlocksSummariesTableQuery(), await this.$checkIfTableExists('blocks_summaries')); + } } catch (e) { throw e; } @@ -512,6 +516,15 @@ class DatabaseMigration { ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`; } + private getCreateBlocksSummariesTableQuery(): string { + return `CREATE TABLE IF NOT EXISTS blocks_summaries ( + height int(10) unsigned NOT NULL, + id varchar(65) NOT NULL, + transactions JSON NOT NULL, + PRIMARY KEY (id) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`; + } + public async $truncateIndexedData(tables: string[]) { const allowedTables = ['blocks', 'hashrates']; diff --git a/backend/src/config.ts b/backend/src/config.ts index e49da3dc9..44864d3b9 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -15,6 +15,7 @@ interface IConfig { INITIAL_BLOCKS_AMOUNT: number; MEMPOOL_BLOCKS_AMOUNT: number; INDEXING_BLOCKS_AMOUNT: number; + BLOCKS_SUMMARIES_INDEXING: boolean; PRICE_FEED_UPDATE_INTERVAL: number; USE_SECOND_NODE_FOR_MINFEE: boolean; EXTERNAL_ASSETS: string[]; @@ -104,6 +105,7 @@ const defaults: IConfig = { 'INITIAL_BLOCKS_AMOUNT': 8, 'MEMPOOL_BLOCKS_AMOUNT': 8, 'INDEXING_BLOCKS_AMOUNT': 11000, // 0 = disable indexing, -1 = index all blocks + 'BLOCKS_SUMMARIES_INDEXING': false, 'PRICE_FEED_UPDATE_INTERVAL': 600, 'USE_SECOND_NODE_FOR_MINFEE': false, 'EXTERNAL_ASSETS': [], diff --git a/backend/src/indexer.ts b/backend/src/indexer.ts index 04c51ffe2..d46991707 100644 --- a/backend/src/indexer.ts +++ b/backend/src/indexer.ts @@ -33,6 +33,7 @@ class Indexer { await this.$resetHashratesIndexingState(); await mining.$generateNetworkHashrateHistory(); await mining.$generatePoolHashrateHistory(); + await blocks.$generateBlocksSummariesDatabase(); } catch (e) { this.reindex(); logger.err(`Indexer failed, trying again later. Reason: ` + (e instanceof Error ? e.message : e)); diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index 4900e3bba..bd39a4494 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -652,6 +652,19 @@ class BlocksRepository { throw e; } } + + /** + * Get a list of blocks that have been indexed + */ + public async $getIndexedBlocks(): Promise { + try { + const [rows]: any = await DB.query(`SELECT height, hash FROM blocks ORDER BY height DESC`); + return rows; + } catch (e) { + logger.err('Cannot generate block size and weight history. Reason: ' + (e instanceof Error ? e.message : e)); + throw e; + } + } } export default new BlocksRepository(); diff --git a/backend/src/repositories/BlocksSummariesRepository.ts b/backend/src/repositories/BlocksSummariesRepository.ts new file mode 100644 index 000000000..73b43ed6f --- /dev/null +++ b/backend/src/repositories/BlocksSummariesRepository.ts @@ -0,0 +1,46 @@ +import DB from '../database'; +import logger from '../logger'; +import { BlockSummary } from '../mempool.interfaces'; + +class BlocksSummariesRepository { + public async $getByBlockId(id: string): Promise { + try { + const [summary]: any[] = await DB.query(`SELECT * from blocks_summaries WHERE id = ?`, [id]); + if (summary.length > 0) { + summary[0].transactions = JSON.parse(summary[0].transactions); + return summary[0]; + } + } catch (e) { + logger.err(`Cannot get block summary for block id ${id}. Reason: ` + (e instanceof Error ? e.message : e)); + } + + return undefined; + } + + public async $saveSummary(height: number, summary: BlockSummary) { + try { + await DB.query(`INSERT INTO blocks_summaries VALUE (?, ?, ?)`, [height, summary.id, JSON.stringify(summary.transactions)]); + } catch (e: any) { + if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart + logger.debug(`Cannot save block summary for ${summary.id} because it has already been indexed, ignoring`); + } else { + logger.debug(`Cannot save block summary for ${summary.id}. Reason: ${e instanceof Error ? e.message : e}`); + throw e; + } + } + } + + public async $getIndexedSummariesId(): Promise { + try { + const [rows]: any[] = await DB.query(`SELECT id from blocks_summaries`); + return rows.map(row => row.id); + } catch (e) { + logger.err(`Cannot get block summaries id list. Reason: ` + (e instanceof Error ? e.message : e)); + } + + return []; + } +} + +export default new BlocksSummariesRepository(); + diff --git a/docker/backend/mempool-config.json b/docker/backend/mempool-config.json index 9a929a4f0..daa4fddc3 100644 --- a/docker/backend/mempool-config.json +++ b/docker/backend/mempool-config.json @@ -19,7 +19,8 @@ "EXTERNAL_RETRY_INTERVAL": __MEMPOOL_EXTERNAL_RETRY_INTERVAL__, "USER_AGENT": "__MEMPOOL_USER_AGENT__", "STDOUT_LOG_MIN_PRIORITY": "__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__", - "INDEXING_BLOCKS_AMOUNT": __MEMPOOL_INDEXING_BLOCKS_AMOUNT__ + "INDEXING_BLOCKS_AMOUNT": __MEMPOOL_INDEXING_BLOCKS_AMOUNT__, + "BLOCKS_SUMMARIES_INDEXING": __MEMPOOL_BLOCKS_SUMMARIES_INDEXING__ }, "CORE_RPC": { "HOST": "__CORE_RPC_HOST__", diff --git a/docker/backend/start.sh b/docker/backend/start.sh index 5792da008..5c4213a1c 100644 --- a/docker/backend/start.sh +++ b/docker/backend/start.sh @@ -14,6 +14,7 @@ __MEMPOOL_BLOCK_WEIGHT_UNITS__=${MEMPOOL_BLOCK_WEIGHT_UNITS:=4000000} __MEMPOOL_INITIAL_BLOCKS_AMOUNT__=${MEMPOOL_INITIAL_BLOCKS_AMOUNT:=8} __MEMPOOL_MEMPOOL_BLOCKS_AMOUNT__=${MEMPOOL_MEMPOOL_BLOCKS_AMOUNT:=8} __MEMPOOL_INDEXING_BLOCKS_AMOUNT__=${MEMPOOL_INDEXING_BLOCKS_AMOUNT:=11000} +__MEMPOOL_BLOCKS_SUMMARIES_INDEXING__=${MEMPOOL_BLOCKS_SUMMARIES_INDEXING:=false} __MEMPOOL_PRICE_FEED_UPDATE_INTERVAL__=${MEMPOOL_PRICE_FEED_UPDATE_INTERVAL:=600} __MEMPOOL_USE_SECOND_NODE_FOR_MINFEE__=${MEMPOOL_USE_SECOND_NODE_FOR_MINFEE:=false} __MEMPOOL_EXTERNAL_ASSETS__=${MEMPOOL_EXTERNAL_ASSETS:=[]} @@ -101,6 +102,7 @@ sed -i "s/__MEMPOOL_BLOCK_WEIGHT_UNITS__/${__MEMPOOL_BLOCK_WEIGHT_UNITS__}/g" me sed -i "s/__MEMPOOL_INITIAL_BLOCKS_AMOUNT__/${__MEMPOOL_INITIAL_BLOCKS_AMOUNT__}/g" mempool-config.json sed -i "s/__MEMPOOL_MEMPOOL_BLOCKS_AMOUNT__/${__MEMPOOL_MEMPOOL_BLOCKS_AMOUNT__}/g" mempool-config.json sed -i "s/__MEMPOOL_INDEXING_BLOCKS_AMOUNT__/${__MEMPOOL_INDEXING_BLOCKS_AMOUNT__}/g" mempool-config.json +sed -i "s/__MEMPOOL_BLOCKS_SUMMARIES_INDEXING__/${__MEMPOOL_BLOCKS_SUMMARIES_INDEXING__}/g" mempool-config.json sed -i "s/__MEMPOOL_PRICE_FEED_UPDATE_INTERVAL__/${__MEMPOOL_PRICE_FEED_UPDATE_INTERVAL__}/g" mempool-config.json sed -i "s/__MEMPOOL_USE_SECOND_NODE_FOR_MINFEE__/${__MEMPOOL_USE_SECOND_NODE_FOR_MINFEE__}/g" mempool-config.json sed -i "s!__MEMPOOL_EXTERNAL_ASSETS__!${__MEMPOOL_EXTERNAL_ASSETS__}!g" mempool-config.json diff --git a/production/mempool-config.mainnet.json b/production/mempool-config.mainnet.json index 1cfeab20d..4575afdbe 100644 --- a/production/mempool-config.mainnet.json +++ b/production/mempool-config.mainnet.json @@ -9,6 +9,7 @@ "CLEAR_PROTECTION_MINUTES": 5, "POLL_RATE_MS": 1000, "INDEXING_BLOCKS_AMOUNT": -1, + "BLOCKS_SUMMARIES_INDEXING": true, "USE_SECOND_NODE_FOR_MINFEE": true }, "SYSLOG" : {