machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #1330 from nymkappa/feature/index-more-data

Browse Source 
Index more data using getblockstats core RPC
This commit is contained in:
softsimon 2022-03-11 14:05:03 +01:00 committed by GitHub
commit 5caaa1633a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 56 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
backend/src/api/blocks.ts
View File

@ -108,17 +108,14 @@ class Blocks {
blockExtended.extras.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0); blockExtended.extras.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0);
blockExtended.extras.coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]); blockExtended.extras.coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]);
const transactionsTmp = [...transactions]; const stats = await bitcoinClient.getBlockStats(block.id);
transactionsTmp.shift(); const coinbaseRaw: IEsploraApi.Transaction = await bitcoinApi.$getRawTransaction(transactions[0].txid, true);
transactionsTmp.sort((a, b) => b.effectiveFeePerVsize - a.effectiveFeePerVsize); blockExtended.extras.coinbaseRaw = coinbaseRaw.hex;
blockExtended.extras.medianFee = stats.feerate_percentiles[2]; // 50th percentiles
blockExtended.extras.medianFee = transactionsTmp.length > 0 ? blockExtended.extras.feeRange = [stats.minfeerate, stats.feerate_percentiles, stats.maxfeerate].flat();
Common.median(transactionsTmp.map((tx) => tx.effectiveFeePerVsize)) : 0; blockExtended.extras.totalFees = stats.totalfee;
blockExtended.extras.feeRange = transactionsTmp.length > 0 ? blockExtended.extras.avgFee = stats.avgfee;
Common.getFeesInRange(transactionsTmp, 8) : [0, 0]; blockExtended.extras.avgFeeRate = stats.avgfeerate;
blockExtended.extras.totalFees = transactionsTmp.reduce((acc, tx) => {
return acc + tx.fee;
}, 0)
if (Common.indexingEnabled()) { if (Common.indexingEnabled()) {
let pool: PoolTag; let pool: PoolTag;
@ -184,7 +181,6 @@ class Blocks {
} }
this.blockIndexingStarted = true; this.blockIndexingStarted = true;
const startedAt = new Date().getTime() / 1000;
try { try {
let currentBlockHeight = blockchainInfo.blocks; let currentBlockHeight = blockchainInfo.blocks;
@ -201,6 +197,9 @@ class Blocks {
const chunkSize = 10000; const chunkSize = 10000;
let totaIndexed = await blocksRepository.$blockCount(null, null); let totaIndexed = await blocksRepository.$blockCount(null, null);
let indexedThisRun = 0; let indexedThisRun = 0;
const startedAt = new Date().getTime() / 1000;
let timer = new Date().getTime() / 1000;
while (currentBlockHeight >= lastBlockToIndex) { while (currentBlockHeight >= lastBlockToIndex) {
const endBlock = Math.max(0, lastBlockToIndex, currentBlockHeight - chunkSize + 1); const endBlock = Math.max(0, lastBlockToIndex, currentBlockHeight - chunkSize + 1);
@ -219,12 +218,16 @@ class Blocks {
break; break;
} }
++indexedThisRun; ++indexedThisRun;
if (++totaIndexed % 100 === 0 || blockHeight === lastBlockToIndex) { ++totaIndexed;
const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt)); const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - timer));
if (elapsedSeconds > 5 || blockHeight === lastBlockToIndex) {
const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt));
const blockPerSeconds = Math.max(1, Math.round(indexedThisRun / elapsedSeconds)); const blockPerSeconds = Math.max(1, Math.round(indexedThisRun / elapsedSeconds));
const progress = Math.round(totaIndexed / indexingBlockAmount * 100); const progress = Math.round(totaIndexed / indexingBlockAmount * 100);
const timeLeft = Math.round((indexingBlockAmount - totaIndexed) / blockPerSeconds); const timeLeft = Math.round((indexingBlockAmount - totaIndexed) / blockPerSeconds);
logger.debug(`Indexing block #${blockHeight} | ~${blockPerSeconds} blocks/sec | total: ${totaIndexed}/${indexingBlockAmount} (${progress}%) | elapsed: ${elapsedSeconds} seconds | left: ~${timeLeft} seconds`); logger.debug(`Indexing block #${blockHeight} | ~${blockPerSeconds} blocks/sec | total: ${totaIndexed}/${indexingBlockAmount} (${progress}%) | elapsed: ${runningFor} seconds | left: ~${timeLeft} seconds`);
timer = new Date().getTime() / 1000;
indexedThisRun = 0;
} }
const blockHash = await bitcoinApi.$getBlockHash(blockHeight); const blockHash = await bitcoinApi.$getBlockHash(blockHeight);
const block = await bitcoinApi.$getBlock(blockHash); const block = await bitcoinApi.$getBlock(blockHash);

22
backend/src/api/database-migration.ts
View File

@ -6,7 +6,7 @@ import logger from '../logger';
const sleep = (ms: number) => new Promise(res => setTimeout(res, ms)); const sleep = (ms: number) => new Promise(res => setTimeout(res, ms));
class DatabaseMigration { class DatabaseMigration {
private static currentVersion = 10; private static currentVersion = 11;
private queryTimeout = 120000; private queryTimeout = 120000;
private statisticsAddedIndexed = false; private statisticsAddedIndexed = false;
@ -92,13 +92,13 @@ class DatabaseMigration {
await this.$executeQuery(connection, this.getCreateBlocksTableQuery(), await this.$checkIfTableExists('blocks')); await this.$executeQuery(connection, this.getCreateBlocksTableQuery(), await this.$checkIfTableExists('blocks'));
} }
if (databaseSchemaVersion < 5 && isBitcoin === true) { if (databaseSchemaVersion < 5 && isBitcoin === true) {
logger.warn(`'blocks' table has been truncated. Re-indexing from scratch.'`); logger.warn(`'blocks' table has been truncated. Re-indexing from scratch.`);
await this.$executeQuery(connection, 'TRUNCATE blocks;'); // Need to re-index await this.$executeQuery(connection, 'TRUNCATE blocks;'); // Need to re-index
await this.$executeQuery(connection, 'ALTER TABLE blocks ADD `reward` double unsigned NOT NULL DEFAULT "0"'); await this.$executeQuery(connection, 'ALTER TABLE blocks ADD `reward` double unsigned NOT NULL DEFAULT "0"');
} }
if (databaseSchemaVersion < 6 && isBitcoin === true) { if (databaseSchemaVersion < 6 && isBitcoin === true) {
logger.warn(`'blocks' table has been truncated. Re-indexing from scratch.'`); logger.warn(`'blocks' table has been truncated. Re-indexing from scratch.`);
await this.$executeQuery(connection, 'TRUNCATE blocks;'); // Need to re-index await this.$executeQuery(connection, 'TRUNCATE blocks;'); // Need to re-index
// Cleanup original blocks fields type // Cleanup original blocks fields type
await this.$executeQuery(connection, 'ALTER TABLE blocks MODIFY `height` integer unsigned NOT NULL DEFAULT "0"'); await this.$executeQuery(connection, 'ALTER TABLE blocks MODIFY `height` integer unsigned NOT NULL DEFAULT "0"');
@ -125,7 +125,7 @@ class DatabaseMigration {
} }
if (databaseSchemaVersion < 8 && isBitcoin === true) { if (databaseSchemaVersion < 8 && isBitcoin === true) {
logger.warn(`'hashrates' table has been truncated. Re-indexing from scratch.'`); logger.warn(`'hashrates' table has been truncated. Re-indexing from scratch.`);
await this.$executeQuery(connection, 'TRUNCATE hashrates;'); // Need to re-index await this.$executeQuery(connection, 'TRUNCATE hashrates;'); // Need to re-index
await this.$executeQuery(connection, 'ALTER TABLE `hashrates` DROP INDEX `PRIMARY`'); await this.$executeQuery(connection, 'ALTER TABLE `hashrates` DROP INDEX `PRIMARY`');
await this.$executeQuery(connection, 'ALTER TABLE `hashrates` ADD `id` int NOT NULL AUTO_INCREMENT PRIMARY KEY FIRST'); await this.$executeQuery(connection, 'ALTER TABLE `hashrates` ADD `id` int NOT NULL AUTO_INCREMENT PRIMARY KEY FIRST');
@ -134,7 +134,7 @@ class DatabaseMigration {
} }
if (databaseSchemaVersion < 9 && isBitcoin === true) { if (databaseSchemaVersion < 9 && isBitcoin === true) {
logger.warn(`'hashrates' table has been truncated. Re-indexing from scratch.'`); logger.warn(`'hashrates' table has been truncated. Re-indexing from scratch.`);
await this.$executeQuery(connection, 'TRUNCATE hashrates;'); // Need to re-index await this.$executeQuery(connection, 'TRUNCATE hashrates;'); // Need to re-index
await this.$executeQuery(connection, 'ALTER TABLE `state` CHANGE `name` `name` varchar(100)'); await this.$executeQuery(connection, 'ALTER TABLE `state` CHANGE `name` `name` varchar(100)');
await this.$executeQuery(connection, 'ALTER TABLE `hashrates` ADD UNIQUE `hashrate_timestamp_pool_id` (`hashrate_timestamp`, `pool_id`)'); await this.$executeQuery(connection, 'ALTER TABLE `hashrates` ADD UNIQUE `hashrate_timestamp_pool_id` (`hashrate_timestamp`, `pool_id`)');
@ -144,6 +144,18 @@ class DatabaseMigration {
await this.$executeQuery(connection, 'ALTER TABLE `blocks` ADD INDEX `blockTimestamp` (`blockTimestamp`)'); await this.$executeQuery(connection, 'ALTER TABLE `blocks` ADD INDEX `blockTimestamp` (`blockTimestamp`)');
} }
if (databaseSchemaVersion < 11 && isBitcoin === true) {
logger.warn(`'blocks' table has been truncated. Re-indexing from scratch.`);
await this.$executeQuery(connection, 'TRUNCATE blocks;'); // Need to re-index
await this.$executeQuery(connection, `ALTER TABLE blocks
ADD avg_fee INT UNSIGNED NULL,
ADD avg_fee_rate INT UNSIGNED NULL
`);
await this.$executeQuery(connection, 'ALTER TABLE blocks MODIFY `reward` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
await this.$executeQuery(connection, 'ALTER TABLE blocks MODIFY `median_fee` INT UNSIGNED NOT NULL DEFAULT "0"');
await this.$executeQuery(connection, 'ALTER TABLE blocks MODIFY `fees` INT UNSIGNED NOT NULL DEFAULT "0"');
}
connection.release(); connection.release();
} catch (e) { } catch (e) {
connection.release(); connection.release();

7
backend/src/mempool.interfaces.ts
View File

@ -79,7 +79,7 @@ export interface TransactionStripped {
export interface BlockExtension { export interface BlockExtension {
totalFees?: number; totalFees?: number;
medianFee?: number; medianFee?: number; // Actually the median fee rate that we compute ourself
feeRange?: number[]; feeRange?: number[];
reward?: number; reward?: number;
coinbaseTx?: TransactionMinerInfo; coinbaseTx?: TransactionMinerInfo;
@ -87,7 +87,10 @@ export interface BlockExtension {
pool?: { pool?: {
id: number; id: number;
name: string; name: string;
} };
avgFee?: number;
avgFeeRate?: number;
coinbaseRaw?: string;
} }
export interface BlockExtended extends IEsploraApi.Block { export interface BlockExtended extends IEsploraApi.Block {

19
backend/src/repositories/BlocksRepository.ts
View File

@ -16,13 +16,13 @@ class BlocksRepository {
weight, tx_count, coinbase_raw, difficulty, weight, tx_count, coinbase_raw, difficulty,
pool_id, fees, fee_span, median_fee, pool_id, fees, fee_span, median_fee,
reward, version, bits, nonce, reward, version, bits, nonce,
merkle_root, previous_block_hash merkle_root, previous_block_hash, avg_fee, avg_fee_rate
) VALUE ( ) VALUE (
?, ?, FROM_UNIXTIME(?), ?, ?, ?, FROM_UNIXTIME(?), ?,
?, ?, ?, ?, ?, ?, ?, ?,
?, ?, ?, ?, ?, ?, ?, ?,
?, ?, ?, ?, ?, ?, ?, ?,
?, ? ?, ?, ?, ?
)`; )`;
const params: any[] = [ const params: any[] = [
@ -32,21 +32,22 @@ class BlocksRepository {
block.size, block.size,
block.weight, block.weight,
block.tx_count, block.tx_count,
'', block.extras.coinbaseRaw,
block.difficulty, block.difficulty,
block.extras.pool?.id, // Should always be set to something block.extras.pool?.id, // Should always be set to something
0, block.extras.totalFees,
'[]', JSON.stringify(block.extras.feeRange),
block.extras.medianFee ?? 0, block.extras.medianFee,
block.extras.reward ?? 0, block.extras.reward,
block.version, block.version,
block.bits, block.bits,
block.nonce, block.nonce,
block.merkle_root, block.merkle_root,
block.previousblockhash block.previousblockhash,
block.extras.avgFee,
block.extras.avgFeeRate,
]; ];
// logger.debug(query);
await connection.query(query, params); await connection.query(query, params);
connection.release(); connection.release();
} catch (e: any) { } catch (e: any) {