machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Index block summaries in db

Browse Source
This commit is contained in:
nymkappa 2022-06-18 16:48:02 +02:00
parent f75da3cc8f
commit f98b829678
11 changed files with 166 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
backend/mempool-config.sample.json
View File

@ -13,6 +13,7 @@
"INITIAL_BLOCKS_AMOUNT": 8,
"MEMPOOL_BLOCKS_AMOUNT": 8,
"INDEXING_BLOCKS_AMOUNT": 11000,
"BLOCKS_SUMMARIES_INDEXING": false,
"PRICE_FEED_UPDATE_INTERVAL": 600,
"USE_SECOND_NODE_FOR_MINFEE": false,
"EXTERNAL_ASSETS": [],

75
backend/src/api/blocks.ts
View File

@ -20,6 +20,7 @@ import indexer from '../indexer';
import fiatConversion from './fiat-conversion';
import RatesRepository from '../repositories/RatesRepository';
import poolsParser from './pools-parser';
import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository';
class Blocks {
private blocks: BlockExtended[] = [];
@ -242,6 +243,53 @@ class Blocks {
}
}
/**
* [INDEXING] Index all blocks summaries for the block txs visualization
*/
public async $generateBlocksSummariesDatabase() {
if (Common.blocksSummariesIndexingEnabled() === false) {
return;
}
try {
// Get all indexed block hash
const indexedBlocks = await blocksRepository.$getIndexedBlocks();
const indexedBlockSummariesHashes = await BlocksSummariesRepository.$getIndexedSummariesId();
// Logging
let totalIndexed = indexedBlockSummariesHashes.length;
let indexedThisRun = 0;
let timer = new Date().getTime() / 1000;
const startedAt = new Date().getTime() / 1000;
for (const block of indexedBlocks) {
if (indexedBlockSummariesHashes.includes(block.hash)) {
continue;
}
// Logging
const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - timer));
if (elapsedSeconds > 5) {
const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt));
const blockPerSeconds = Math.max(1, indexedThisRun / elapsedSeconds);
const progress = Math.round(totalIndexed / indexedBlocks.length * 10000) / 100;
const timeLeft = Math.round((indexedBlocks.length - totalIndexed) / blockPerSeconds);
logger.debug(`Indexing block summary for #${block.height} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${totalIndexed}/${indexedBlocks.length} (${progress}%) | elapsed: ${runningFor} seconds | left: ~${timeLeft} seconds`);
timer = new Date().getTime() / 1000;
indexedThisRun = 0;
}
await this.$getStrippedBlockTransactions(block.hash, true, true); // This will index the block summary
// Logging
indexedThisRun++;
totalIndexed++;
}
} catch (e) {
logger.err(`Blocks summaries indexing failed. Reason: ${(e instanceof Error ? e.message : e)}`);
}
}
/**
* [INDEXING] Index all blocks metadata for the mining dashboard
*/
@ -292,7 +340,7 @@ class Blocks {
const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - timer));
if (elapsedSeconds > 5 || blockHeight === lastBlockToIndex) {
const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt));
const blockPerSeconds = Math.max(1, Math.round(indexedThisRun / elapsedSeconds));
const blockPerSeconds = Math.max(1, indexedThisRun / elapsedSeconds);
const progress = Math.round(totalIndexed / indexingBlockAmount * 10000) / 100;
const timeLeft = Math.round((indexingBlockAmount - totalIndexed) / blockPerSeconds);
logger.debug(`Indexing block #${blockHeight} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${totalIndexed}/${indexingBlockAmount} (${progress}%) | elapsed: ${runningFor} seconds | left: ~${timeLeft} seconds`);
@ -392,6 +440,11 @@ class Blocks {
}
}
await blocksRepository.$saveBlockInDatabase(blockExtended);
// Save blocks summary for visualization if it's enabled
if (Common.blocksSummariesIndexingEnabled() === true) {
await this.$getStrippedBlockTransactions(blockExtended.id, true);
}
}
}
if (fiatConversion.ratesInitialized === true && config.DATABASE.ENABLED === true) {
@ -477,14 +530,32 @@ class Blocks {
return blockExtended;
}
public async $getStrippedBlockTransactions(hash: string): Promise<TransactionStripped[]> {
public async $getStrippedBlockTransactions(hash: string, skipMemoryCache: boolean = false, skipDBLookup: boolean = false): Promise<TransactionStripped[]> {
if (skipMemoryCache === false) {
// Check the memory cache
const cachedSummary = this.getBlockSummaries().find((b) => b.id === hash);
if (cachedSummary) {
return cachedSummary.transactions;
}
}
// Check if it's indexed in db
if (skipDBLookup === false && Common.blocksSummariesIndexingEnabled() === true) {
const indexedSummary = await BlocksSummariesRepository.$getByBlockId(hash);
if (indexedSummary !== undefined) {
return indexedSummary.transactions;
}
}
// Call Core RPC
const block = await bitcoinClient.getBlock(hash, 2);
const summary = this.summarizeBlock(block);
// Index the response if needed
if (Common.blocksSummariesIndexingEnabled() === true) {
await BlocksSummariesRepository.$saveSummary(block.height, summary);
}
return summary.transactions;
}

7
backend/src/api/common.ts
View File

@ -177,4 +177,11 @@ export class Common {
config.MEMPOOL.INDEXING_BLOCKS_AMOUNT !== 0
);
}
static blocksSummariesIndexingEnabled(): boolean {
return (
Common.indexingEnabled() &&
config.MEMPOOL.BLOCKS_SUMMARIES_INDEXING === true
);
}
}

15
backend/src/api/database-migration.ts
View File

@ -4,7 +4,7 @@ import logger from '../logger';
import { Common } from './common';
class DatabaseMigration {
private static currentVersion = 19;
private static currentVersion = 20;
private queryTimeout = 120000;
private statisticsAddedIndexed = false;
private uniqueLogs: string[] = [];
@ -217,6 +217,10 @@ class DatabaseMigration {
if (databaseSchemaVersion < 19) {
await this.$executeQuery(this.getCreateRatesTableQuery(), await this.$checkIfTableExists('rates'));
}
if (databaseSchemaVersion < 20 && isBitcoin === true) {
await this.$executeQuery(this.getCreateBlocksSummariesTableQuery(), await this.$checkIfTableExists('blocks_summaries'));
}
} catch (e) {
throw e;
}
@ -512,6 +516,15 @@ class DatabaseMigration {
) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
}
private getCreateBlocksSummariesTableQuery(): string {
return `CREATE TABLE IF NOT EXISTS blocks_summaries (
height int(10) unsigned NOT NULL,
id varchar(65) NOT NULL,
transactions JSON NOT NULL,
PRIMARY KEY (id)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
}
public async $truncateIndexedData(tables: string[]) {
const allowedTables = ['blocks', 'hashrates'];

2
backend/src/config.ts
View File

@ -15,6 +15,7 @@ interface IConfig {
INITIAL_BLOCKS_AMOUNT: number;
MEMPOOL_BLOCKS_AMOUNT: number;
INDEXING_BLOCKS_AMOUNT: number;
BLOCKS_SUMMARIES_INDEXING: boolean;
PRICE_FEED_UPDATE_INTERVAL: number;
USE_SECOND_NODE_FOR_MINFEE: boolean;
EXTERNAL_ASSETS: string[];
@ -104,6 +105,7 @@ const defaults: IConfig = {
'INITIAL_BLOCKS_AMOUNT': 8,
'MEMPOOL_BLOCKS_AMOUNT': 8,
'INDEXING_BLOCKS_AMOUNT': 11000, // 0 = disable indexing, -1 = index all blocks
'BLOCKS_SUMMARIES_INDEXING': false,
'PRICE_FEED_UPDATE_INTERVAL': 600,
'USE_SECOND_NODE_FOR_MINFEE': false,
'EXTERNAL_ASSETS': [],

1
backend/src/indexer.ts
View File

@ -33,6 +33,7 @@ class Indexer {
await this.$resetHashratesIndexingState();
await mining.$generateNetworkHashrateHistory();
await mining.$generatePoolHashrateHistory();
await blocks.$generateBlocksSummariesDatabase();
} catch (e) {
this.reindex();
logger.err(`Indexer failed, trying again later. Reason: ` + (e instanceof Error ? e.message : e));

13
backend/src/repositories/BlocksRepository.ts
View File

@ -652,6 +652,19 @@ class BlocksRepository {
throw e;
}
}
/**
* Get a list of blocks that have been indexed
*/
public async $getIndexedBlocks(): Promise<any[]> {
try {
const [rows]: any = await DB.query(`SELECT height, hash FROM blocks ORDER BY height DESC`);
return rows;
} catch (e) {
logger.err('Cannot generate block size and weight history. Reason: ' + (e instanceof Error ? e.message : e));
throw e;
}
}
}
export default new BlocksRepository();

46
backend/src/repositories/BlocksSummariesRepository.ts Normal file
View File

@ -0,0 +1,46 @@
import DB from '../database';
import logger from '../logger';
import { BlockSummary } from '../mempool.interfaces';
class BlocksSummariesRepository {
public async $getByBlockId(id: string): Promise<BlockSummary | undefined> {
try {
const [summary]: any[] = await DB.query(`SELECT * from blocks_summaries WHERE id = ?`, [id]);
if (summary.length > 0) {
summary[0].transactions = JSON.parse(summary[0].transactions);
return summary[0];
}
} catch (e) {
logger.err(`Cannot get block summary for block id ${id}. Reason: ` + (e instanceof Error ? e.message : e));
}
return undefined;
}
public async $saveSummary(height: number, summary: BlockSummary) {
try {
await DB.query(`INSERT INTO blocks_summaries VALUE (?, ?, ?)`, [height, summary.id, JSON.stringify(summary.transactions)]);
} catch (e: any) {
if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
logger.debug(`Cannot save block summary for ${summary.id} because it has already been indexed, ignoring`);
} else {
logger.debug(`Cannot save block summary for ${summary.id}. Reason: ${e instanceof Error ? e.message : e}`);
throw e;
}
}
}
public async $getIndexedSummariesId(): Promise<string[]> {
try {
const [rows]: any[] = await DB.query(`SELECT id from blocks_summaries`);
return rows.map(row => row.id);
} catch (e) {
logger.err(`Cannot get block summaries id list. Reason: ` + (e instanceof Error ? e.message : e));
}
return [];
}
}
export default new BlocksSummariesRepository();

3
docker/backend/mempool-config.json
View File

@ -19,7 +19,8 @@
"EXTERNAL_RETRY_INTERVAL": __MEMPOOL_EXTERNAL_RETRY_INTERVAL__,
"USER_AGENT": "__MEMPOOL_USER_AGENT__",
"STDOUT_LOG_MIN_PRIORITY": "__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__",
"INDEXING_BLOCKS_AMOUNT": __MEMPOOL_INDEXING_BLOCKS_AMOUNT__
"INDEXING_BLOCKS_AMOUNT": __MEMPOOL_INDEXING_BLOCKS_AMOUNT__,
"BLOCKS_SUMMARIES_INDEXING": __MEMPOOL_BLOCKS_SUMMARIES_INDEXING__
},
"CORE_RPC": {
"HOST": "__CORE_RPC_HOST__",

2
docker/backend/start.sh
View File

@ -14,6 +14,7 @@ __MEMPOOL_BLOCK_WEIGHT_UNITS__=${MEMPOOL_BLOCK_WEIGHT_UNITS:=4000000}
__MEMPOOL_INITIAL_BLOCKS_AMOUNT__=${MEMPOOL_INITIAL_BLOCKS_AMOUNT:=8}
__MEMPOOL_MEMPOOL_BLOCKS_AMOUNT__=${MEMPOOL_MEMPOOL_BLOCKS_AMOUNT:=8}
__MEMPOOL_INDEXING_BLOCKS_AMOUNT__=${MEMPOOL_INDEXING_BLOCKS_AMOUNT:=11000}
__MEMPOOL_BLOCKS_SUMMARIES_INDEXING__=${MEMPOOL_BLOCKS_SUMMARIES_INDEXING:=false}
__MEMPOOL_PRICE_FEED_UPDATE_INTERVAL__=${MEMPOOL_PRICE_FEED_UPDATE_INTERVAL:=600}
__MEMPOOL_USE_SECOND_NODE_FOR_MINFEE__=${MEMPOOL_USE_SECOND_NODE_FOR_MINFEE:=false}
__MEMPOOL_EXTERNAL_ASSETS__=${MEMPOOL_EXTERNAL_ASSETS:=[]}
@ -101,6 +102,7 @@ sed -i "s/__MEMPOOL_BLOCK_WEIGHT_UNITS__/${__MEMPOOL_BLOCK_WEIGHT_UNITS__}/g" me
sed -i "s/__MEMPOOL_INITIAL_BLOCKS_AMOUNT__/${__MEMPOOL_INITIAL_BLOCKS_AMOUNT__}/g" mempool-config.json
sed -i "s/__MEMPOOL_MEMPOOL_BLOCKS_AMOUNT__/${__MEMPOOL_MEMPOOL_BLOCKS_AMOUNT__}/g" mempool-config.json
sed -i "s/__MEMPOOL_INDEXING_BLOCKS_AMOUNT__/${__MEMPOOL_INDEXING_BLOCKS_AMOUNT__}/g" mempool-config.json
sed -i "s/__MEMPOOL_BLOCKS_SUMMARIES_INDEXING__/${__MEMPOOL_BLOCKS_SUMMARIES_INDEXING__}/g" mempool-config.json
sed -i "s/__MEMPOOL_PRICE_FEED_UPDATE_INTERVAL__/${__MEMPOOL_PRICE_FEED_UPDATE_INTERVAL__}/g" mempool-config.json
sed -i "s/__MEMPOOL_USE_SECOND_NODE_FOR_MINFEE__/${__MEMPOOL_USE_SECOND_NODE_FOR_MINFEE__}/g" mempool-config.json
sed -i "s!__MEMPOOL_EXTERNAL_ASSETS__!${__MEMPOOL_EXTERNAL_ASSETS__}!g" mempool-config.json

1
production/mempool-config.mainnet.json
View File

@ -9,6 +9,7 @@
"CLEAR_PROTECTION_MINUTES": 5,
"POLL_RATE_MS": 1000,
"INDEXING_BLOCKS_AMOUNT": -1,
"BLOCKS_SUMMARIES_INDEXING": true,
"USE_SECOND_NODE_FOR_MINFEE": true
},
"SYSLOG" : {