machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #1235 from nymkappa/feature/blocks-extras

Browse Source 
Added /api/v1/blocks-extras endpoint
This commit is contained in:
wiz 2022-02-12 11:30:49 +00:00 committed by GitHub
commit 039a627d1c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 199 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

178
backend/src/api/blocks.ts
View File

@ -10,6 +10,7 @@ import bitcoinClient from './bitcoin/bitcoin-client';
import { IEsploraApi } from './bitcoin/esplora-api.interface';
import poolsRepository from '../repositories/PoolsRepository';
import blocksRepository from '../repositories/BlocksRepository';
import loadingIndicators from './loading-indicators';
class Blocks {
private blocks: BlockExtended[] = [];
@ -41,7 +42,12 @@ class Blocks {
* @param onlyCoinbase - Set to true if you only need the coinbase transaction
* @returns Promise<TransactionExtended[]>
*/
private async $getTransactionsExtended(blockHash: string, blockHeight: number, onlyCoinbase: boolean): Promise<TransactionExtended[]> {
private async $getTransactionsExtended(
blockHash: string,
blockHeight: number,
onlyCoinbase: boolean,
quiet: boolean = false,
): Promise<TransactionExtended[]> {
const transactions: TransactionExtended[] = [];
const txIds: string[] = await bitcoinApi.$getTxIdsForBlock(blockHash);
@ -57,7 +63,7 @@ class Blocks {
transactionsFound++;
} else if (config.MEMPOOL.BACKEND === 'esplora' || memPool.isInSync() || i === 0) {
// Otherwise we fetch the tx data through backend services (esplora, electrum, core rpc...)
if (i % (Math.round((txIds.length) / 10)) === 0 || i + 1 === txIds.length) { // Avoid log spam
if (!quiet && (i % (Math.round((txIds.length) / 10)) === 0 || i + 1 === txIds.length)) { // Avoid log spam
logger.debug(`Indexing tx ${i + 1} of ${txIds.length} in block #${blockHeight}`);
}
try {
@ -83,7 +89,9 @@ class Blocks {
}
});
logger.debug(`${transactionsFound} of ${txIds.length} found in mempool. ${transactionsFetched} fetched through backend service.`);
if (!quiet) {
logger.debug(`${transactionsFound} of ${txIds.length} found in mempool. ${transactionsFetched} fetched through backend service.`);
}
return transactions;
}
@ -94,13 +102,10 @@ class Blocks {
* @param transactions
* @returns BlockExtended
*/
private getBlockExtended(block: IEsploraApi.Block, transactions: TransactionExtended[]): BlockExtended {
const blockExtended: BlockExtended = Object.assign({}, block);
blockExtended.extras = {
reward: transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0),
coinbaseTx: transactionUtils.stripCoinbaseTransaction(transactions[0]),
};
private async $getBlockExtended(block: IEsploraApi.Block, transactions: TransactionExtended[]): Promise<BlockExtended> {
const blockExtended: BlockExtended = Object.assign({extras: {}}, block);
blockExtended.extras.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0);
blockExtended.extras.coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]);
const transactionsTmp = [...transactions];
transactionsTmp.shift();
@ -111,6 +116,22 @@ class Blocks {
blockExtended.extras.feeRange = transactionsTmp.length > 0 ?
Common.getFeesInRange(transactionsTmp, 8) : [0, 0];
const indexingAvailable =
['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) &&
config.DATABASE.ENABLED === true;
if (indexingAvailable) {
let pool: PoolTag;
if (blockExtended.extras?.coinbaseTx !== undefined) {
pool = await this.$findBlockMiner(blockExtended.extras?.coinbaseTx);
} else {
pool = await poolsRepository.$getUnknownPool();
}
blockExtended.extras.pool = {
id: pool.id,
name: pool.name
};
}
return blockExtended;
}
@ -153,19 +174,21 @@ class Blocks {
*/
public async $generateBlockDatabase() {
if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false || // Bitcoin only
config.MEMPOOL.INDEXING_BLOCKS_AMOUNT === 0 || // Indexing must be enabled
config.MEMPOOL.INDEXING_BLOCKS_AMOUNT === 0 || // Indexing of older blocks must be enabled
!memPool.isInSync() || // We sync the mempool first
this.blockIndexingStarted === true // Indexing must not already be in progress
this.blockIndexingStarted === true || // Indexing must not already be in progress
config.DATABASE.ENABLED === false
) {
return;
}
const blockchainInfo = await bitcoinClient.getBlockchainInfo();
if (blockchainInfo.blocks !== blockchainInfo.headers) {
if (blockchainInfo.blocks !== blockchainInfo.headers) { // Wait for node to sync
return;
}
this.blockIndexingStarted = true;
const startedAt = new Date().getTime() / 1000;
try {
let currentBlockHeight = blockchainInfo.blocks;
@ -180,6 +203,7 @@ class Blocks {
logger.info(`Indexing blocks from #${currentBlockHeight} to #${lastBlockToIndex}`);
const chunkSize = 10000;
let totaIndexed = 0;
while (currentBlockHeight >= lastBlockToIndex) {
const endBlock = Math.max(0, lastBlockToIndex, currentBlockHeight - chunkSize + 1);
@ -198,21 +222,17 @@ class Blocks {
break;
}
try {
logger.debug(`Indexing block #${blockHeight}`);
if (totaIndexed % 100 === 0 || blockHeight === lastBlockToIndex) {
const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt));
const blockPerSeconds = Math.round(totaIndexed / elapsedSeconds);
logger.debug(`Indexing block #${blockHeight} | ~${blockPerSeconds} blocks/sec | total: ${totaIndexed} | elapsed: ${elapsedSeconds} seconds`);
}
const blockHash = await bitcoinApi.$getBlockHash(blockHeight);
const block = await bitcoinApi.$getBlock(blockHash);
const transactions = await this.$getTransactionsExtended(blockHash, block.height, true);
const blockExtended = this.getBlockExtended(block, transactions);
let miner: PoolTag;
if (blockExtended?.extras?.coinbaseTx) {
miner = await this.$findBlockMiner(blockExtended.extras.coinbaseTx);
} else {
miner = await poolsRepository.$getUnknownPool();
}
const coinbase: IEsploraApi.Transaction = await bitcoinApi.$getRawTransaction(transactions[0].txid, true);
await blocksRepository.$saveBlockInDatabase(blockExtended, blockHash, coinbase.hex, miner);
const transactions = await this.$getTransactionsExtended(blockHash, block.height, true, true);
const blockExtended = await this.$getBlockExtended(block, transactions);
await blocksRepository.$saveBlockInDatabase(blockExtended);
++totaIndexed;
} catch (e) {
logger.err(`Something went wrong while indexing blocks.` + e);
}
@ -271,17 +291,13 @@ class Blocks {
const block = await bitcoinApi.$getBlock(blockHash);
const txIds: string[] = await bitcoinApi.$getTxIdsForBlock(blockHash);
const transactions = await this.$getTransactionsExtended(blockHash, block.height, false);
const blockExtended: BlockExtended = this.getBlockExtended(block, transactions);
const coinbase: IEsploraApi.Transaction = await bitcoinApi.$getRawTransaction(transactions[0].txid, true);
const blockExtended: BlockExtended = await this.$getBlockExtended(block, transactions);
if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === true) {
let miner: PoolTag;
if (blockExtended?.extras?.coinbaseTx) {
miner = await this.$findBlockMiner(blockExtended.extras.coinbaseTx);
} else {
miner = await poolsRepository.$getUnknownPool();
}
await blocksRepository.$saveBlockInDatabase(blockExtended, blockHash, coinbase.hex, miner);
const indexingAvailable =
['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) &&
config.DATABASE.ENABLED === true;
if (indexingAvailable) {
await blocksRepository.$saveBlockInDatabase(blockExtended);
}
if (block.height % 2016 === 0) {
@ -304,6 +320,96 @@ class Blocks {
}
}
/**
* Index a block if it's missing from the database. Returns the block after indexing
*/
public async $indexBlock(height: number): Promise<BlockExtended> {
const dbBlock = await blocksRepository.$getBlockByHeight(height);
if (dbBlock != null) {
return this.prepareBlock(dbBlock);
}
const blockHash = await bitcoinApi.$getBlockHash(height);
const block = await bitcoinApi.$getBlock(blockHash);
const transactions = await this.$getTransactionsExtended(blockHash, block.height, true);
const blockExtended = await this.$getBlockExtended(block, transactions);
await blocksRepository.$saveBlockInDatabase(blockExtended);
return blockExtended;
}
public async $getBlocksExtras(fromHeight: number): Promise<BlockExtended[]> {
const indexingAvailable =
['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) &&
config.DATABASE.ENABLED === true;
try {
loadingIndicators.setProgress('blocks', 0);
let currentHeight = fromHeight ? fromHeight : this.getCurrentBlockHeight();
const returnBlocks: BlockExtended[] = [];
if (currentHeight < 0) {
return returnBlocks;
}
// Check if block height exist in local cache to skip the hash lookup
const blockByHeight = this.getBlocks().find((b) => b.height === currentHeight);
let startFromHash: string | null = null;
if (blockByHeight) {
startFromHash = blockByHeight.id;
} else {
startFromHash = await bitcoinApi.$getBlockHash(currentHeight);
}
let nextHash = startFromHash;
for (let i = 0; i < 10 && currentHeight >= 0; i++) {
let block = this.getBlocks().find((b) => b.height === currentHeight);
if (!block && indexingAvailable) {
block = this.prepareBlock(await this.$indexBlock(currentHeight));
} else if (!block) {
block = this.prepareBlock(await bitcoinApi.$getBlock(nextHash));
}
returnBlocks.push(block);
nextHash = block.previousblockhash;
loadingIndicators.setProgress('blocks', i / 10 * 100);
currentHeight--;
}
return returnBlocks;
} catch (e) {
loadingIndicators.setProgress('blocks', 100);
throw e;
}
}
private prepareBlock(block: any): BlockExtended {
return <BlockExtended>{
id: block.id ?? block.hash, // hash for indexed block
timestamp: block?.timestamp ?? block?.blockTimestamp, // blockTimestamp for indexed block
height: block?.height,
version: block?.version,
bits: block?.bits,
nonce: block?.nonce,
difficulty: block?.difficulty,
merkle_root: block?.merkle_root,
tx_count: block?.tx_count,
size: block?.size,
weight: block?.weight,
previousblockhash: block?.previousblockhash,
extras: {
medianFee: block?.medianFee,
feeRange: block?.feeRange ?? [], // TODO
reward: block?.reward,
pool: block?.extras?.pool ?? (block?.pool_id ? {
id: block?.pool_id,
name: block?.pool_name,
} : undefined),
}
};
}
public getLastDifficultyAdjustmentTime(): number {
return this.lastDifficultyAdjustmentTime;
}

6
backend/src/api/database-migration.ts
View File

@ -6,7 +6,7 @@ import logger from '../logger';
const sleep = (ms: number) => new Promise(res => setTimeout(res, ms));
class DatabaseMigration {
private static currentVersion = 4;
private static currentVersion = 5;
private queryTimeout = 120000;
private statisticsAddedIndexed = false;
@ -229,6 +229,10 @@ class DatabaseMigration {
}
}
if (version < 5 && (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === true)) {
queries.push('ALTER TABLE blocks ADD `reward` double unsigned NOT NULL DEFAULT "0"');
}
return queries;
}

4
backend/src/index.ts
View File

@ -290,6 +290,10 @@ class Server {
;
}
this.app
.get(config.MEMPOOL.API_URL_PREFIX + 'blocks-extras', routes.getBlocksExtras)
.get(config.MEMPOOL.API_URL_PREFIX + 'blocks-extras/:height', routes.getBlocksExtras);
if (config.MEMPOOL.BACKEND !== 'esplora') {
this.app
.get(config.MEMPOOL.API_URL_PREFIX + 'mempool', routes.getMempool)

8
backend/src/mempool.interfaces.ts
View File

@ -1,7 +1,7 @@
import { IEsploraApi } from './api/bitcoin/esplora-api.interface';
export interface PoolTag {
id: number | null, // mysql row id
id: number, // mysql row id
name: string,
link: string,
regexes: string, // JSON array
@ -83,10 +83,14 @@ export interface BlockExtension {
reward?: number;
coinbaseTx?: TransactionMinerInfo;
matchRate?: number;
pool?: {
id: number;
name: string;
}
}
export interface BlockExtended extends IEsploraApi.Block {
extras?: BlockExtension;
extras: BlockExtension;
}
export interface TransactionMinerInfo {

42
backend/src/repositories/BlocksRepository.ts
View File

@ -11,38 +11,36 @@ class BlocksRepository {
/**
* Save indexed block data in the database
*/
public async $saveBlockInDatabase(
block: BlockExtended,
blockHash: string,
coinbaseHex: string | undefined,
poolTag: PoolTag
) {
public async $saveBlockInDatabase(block: BlockExtended) {
const connection = await DB.pool.getConnection();
try {
const query = `INSERT INTO blocks(
height, hash, blockTimestamp, size,
weight, tx_count, coinbase_raw, difficulty,
pool_id, fees, fee_span, median_fee
pool_id, fees, fee_span, median_fee,
reward
) VALUE (
?, ?, FROM_UNIXTIME(?), ?,
?, ?, ?, ?,
?, ?, ?, ?
?, ?, ?, ?,
?
)`;
const params: any[] = [
block.height,
blockHash,
block.id,
block.timestamp,
block.size,
block.weight,
block.tx_count,
coinbaseHex ? coinbaseHex : '',
'',
block.difficulty,
poolTag.id,
block.extras?.pool?.id, // Should always be set to something
0,
'[]',
block.extras ? block.extras.medianFee : 0,
block.extras.medianFee ?? 0,
block.extras?.reward ?? 0,
];
await connection.query(query, params);
@ -136,6 +134,26 @@ class BlocksRepository {
return <number>rows[0].blockTimestamp;
}
/**
* Get one block by height
*/
public async $getBlockByHeight(height: number): Promise<object | null> {
const connection = await DB.pool.getConnection();
const [rows]: any[] = await connection.query(`
SELECT *, UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp, pools.id as pool_id, pools.name as pool_name, pools.link as pool_link, pools.addresses as pool_addresses, pools.regexes as pool_regexes
FROM blocks
JOIN pools ON blocks.pool_id = pools.id
WHERE height = ${height};
`);
connection.release();
if (rows.length <= 0) {
return null;
}
return rows[0];
}
}
export default new BlocksRepository();

4
backend/src/repositories/PoolsRepository.ts
View File

@ -7,7 +7,7 @@ class PoolsRepository {
*/
public async $getPools(): Promise<PoolTag[]> {
const connection = await DB.pool.getConnection();
const [rows] = await connection.query('SELECT * FROM pools;');
const [rows] = await connection.query('SELECT id, name, addresses, regexes FROM pools;');
connection.release();
return <PoolTag[]>rows;
}
@ -17,7 +17,7 @@ class PoolsRepository {
*/
public async $getUnknownPool(): Promise<PoolTag> {
const connection = await DB.pool.getConnection();
const [rows] = await connection.query('SELECT * FROM pools where name = "Unknown"');
const [rows] = await connection.query('SELECT id, name FROM pools where name = "Unknown"');
connection.release();
return <PoolTag>rows[0];
}

8
backend/src/routes.ts
View File

@ -564,6 +564,14 @@ class Routes {
}
}
public async getBlocksExtras(req: Request, res: Response) {
try {
res.json(await blocks.$getBlocksExtras(parseInt(req.params.height, 10)))
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
}
}
public async getBlocks(req: Request, res: Response) {
try {
loadingIndicators.setProgress('blocks', 0);

3
docker/backend/mempool-config.json
View File

@ -15,7 +15,8 @@
"PRICE_FEED_UPDATE_INTERVAL": __MEMPOOL_PRICE_FEED_UPDATE_INTERVAL__,
"USE_SECOND_NODE_FOR_MINFEE": __MEMPOOL_USE_SECOND_NODE_FOR_MINFEE__,
"EXTERNAL_ASSETS": __MEMPOOL_EXTERNAL_ASSETS__,
"STDOUT_LOG_MIN_PRIORITY": "__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__"
"STDOUT_LOG_MIN_PRIORITY": "__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__",
"INDEXING_BLOCKS_AMOUNT": __MEMPOOL_INDEXING_BLOCKS_AMOUNT__
},
"CORE_RPC": {
"HOST": "__CORE_RPC_HOST__",