machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Enforce BlockExtended use for block indexing - Unify /api/v1/block(s) API(s) response format

Browse Source
This commit is contained in:
nymkappa 2023-02-27 18:00:00 +09:00
parent a68deda4fe
commit 8b947a6c67
12 changed files with 352 additions and 239 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
backend/src/api/bitcoin/bitcoin-api.interface.ts
View File

@ -172,4 +172,35 @@ export namespace IBitcoinApi {
}
}
export interface BlockStats {
"avgfee": number;
"avgfeerate": number;
"avgtxsize": number;
"blockhash": string;
"feerate_percentiles": [number, number, number, number, number];
"height": number;
"ins": number;
"maxfee": number;
"maxfeerate": number;
"maxtxsize": number;
"medianfee": number;
"mediantime": number;
"mediantxsize": number;
"minfee": number;
"minfeerate": number;
"mintxsize": number;
"outs": number;
"subsidy": number;
"swtotal_size": number;
"swtotal_weight": number;
"swtxs": number;
"time": number;
"total_out": number;
"total_size": number;
"total_weight": number;
"totalfee": number;
"txs": number;
"utxo_increase": number;
"utxo_size_inc": number;
}
}

2
backend/src/api/bitcoin/bitcoin-api.ts
View File

@ -28,7 +28,7 @@ class BitcoinApi implements AbstractBitcoinApi {
size: block.size,
weight: block.weight,
previousblockhash: block.previousblockhash,
medianTime: block.mediantime,
mediantime: block.mediantime,
};
}

2
backend/src/api/bitcoin/esplora-api.interface.ts
View File

@ -88,7 +88,7 @@ export namespace IEsploraApi {
size: number;
weight: number;
previousblockhash: string;
medianTime?: number;
mediantime: number;
}
export interface Address {

224
backend/src/api/blocks.ts
View File

@ -2,7 +2,7 @@ import config from '../config';
import bitcoinApi from './bitcoin/bitcoin-api-factory';
import logger from '../logger';
import memPool from './mempool';
import { BlockExtended, BlockSummary, PoolTag, TransactionExtended, TransactionStripped, TransactionMinerInfo } from '../mempool.interfaces';
import { BlockExtended, BlockExtension, BlockSummary, PoolTag, TransactionExtended, TransactionStripped, TransactionMinerInfo } from '../mempool.interfaces';
import { Common } from './common';
import diskCache from './disk-cache';
import transactionUtils from './transaction-utils';
@ -13,7 +13,6 @@ import poolsRepository from '../repositories/PoolsRepository';
import blocksRepository from '../repositories/BlocksRepository';
import loadingIndicators from './loading-indicators';
import BitcoinApi from './bitcoin/bitcoin-api';
import { prepareBlock } from '../utils/blocks-utils';
import BlocksRepository from '../repositories/BlocksRepository';
import HashratesRepository from '../repositories/HashratesRepository';
import indexer from '../indexer';
@ -143,7 +142,7 @@ class Blocks {
* @param block
* @returns BlockSummary
*/
private summarizeBlock(block: IBitcoinApi.VerboseBlock): BlockSummary {
public summarizeBlock(block: IBitcoinApi.VerboseBlock): BlockSummary {
const stripped = block.tx.map((tx) => {
return {
txid: tx.txid,
@ -166,80 +165,81 @@ class Blocks {
* @returns BlockExtended
*/
private async $getBlockExtended(block: IEsploraApi.Block, transactions: TransactionExtended[]): Promise<BlockExtended> {
const blk: BlockExtended = Object.assign({ extras: {} }, block);
blk.extras.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0);
blk.extras.coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]);
blk.extras.coinbaseRaw = blk.extras.coinbaseTx.vin[0].scriptsig;
blk.extras.usd = priceUpdater.latestPrices.USD;
blk.extras.medianTimestamp = block.medianTime;
blk.extras.orphans = chainTips.getOrphanedBlocksAtHeight(blk.height);
const coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]);
const blk: Partial<BlockExtended> = Object.assign({}, block);
const extras: Partial<BlockExtension> = {};
extras.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0);
extras.coinbaseRaw = coinbaseTx.vin[0].scriptsig;
extras.orphans = chainTips.getOrphanedBlocksAtHeight(blk.height);
if (block.height === 0) {
blk.extras.medianFee = 0; // 50th percentiles
blk.extras.feeRange = [0, 0, 0, 0, 0, 0, 0];
blk.extras.totalFees = 0;
blk.extras.avgFee = 0;
blk.extras.avgFeeRate = 0;
blk.extras.utxoSetChange = 0;
blk.extras.avgTxSize = 0;
blk.extras.totalInputs = 0;
blk.extras.totalOutputs = 1;
blk.extras.totalOutputAmt = 0;
blk.extras.segwitTotalTxs = 0;
blk.extras.segwitTotalSize = 0;
blk.extras.segwitTotalWeight = 0;
extras.medianFee = 0; // 50th percentiles
extras.feeRange = [0, 0, 0, 0, 0, 0, 0];
extras.totalFees = 0;
extras.avgFee = 0;
extras.avgFeeRate = 0;
extras.utxoSetChange = 0;
extras.avgTxSize = 0;
extras.totalInputs = 0;
extras.totalOutputs = 1;
extras.totalOutputAmt = 0;
extras.segwitTotalTxs = 0;
extras.segwitTotalSize = 0;
extras.segwitTotalWeight = 0;
} else {
const stats = await bitcoinClient.getBlockStats(block.id);
blk.extras.medianFee = stats.feerate_percentiles[2]; // 50th percentiles
blk.extras.feeRange = [stats.minfeerate, stats.feerate_percentiles, stats.maxfeerate].flat();
blk.extras.totalFees = stats.totalfee;
blk.extras.avgFee = stats.avgfee;
blk.extras.avgFeeRate = stats.avgfeerate;
blk.extras.utxoSetChange = stats.utxo_increase;
blk.extras.avgTxSize = Math.round(stats.total_size / stats.txs * 100) * 0.01;
blk.extras.totalInputs = stats.ins;
blk.extras.totalOutputs = stats.outs;
blk.extras.totalOutputAmt = stats.total_out;
blk.extras.segwitTotalTxs = stats.swtxs;
blk.extras.segwitTotalSize = stats.swtotal_size;
blk.extras.segwitTotalWeight = stats.swtotal_weight;
const stats: IBitcoinApi.BlockStats = await bitcoinClient.getBlockStats(block.id);
extras.medianFee = stats.feerate_percentiles[2]; // 50th percentiles
extras.feeRange = [stats.minfeerate, stats.feerate_percentiles, stats.maxfeerate].flat();
extras.totalFees = stats.totalfee;
extras.avgFee = stats.avgfee;
extras.avgFeeRate = stats.avgfeerate;
extras.utxoSetChange = stats.utxo_increase;
extras.avgTxSize = Math.round(stats.total_size / stats.txs * 100) * 0.01;
extras.totalInputs = stats.ins;
extras.totalOutputs = stats.outs;
extras.totalOutputAmt = stats.total_out;
extras.segwitTotalTxs = stats.swtxs;
extras.segwitTotalSize = stats.swtotal_size;
extras.segwitTotalWeight = stats.swtotal_weight;
}
if (Common.blocksSummariesIndexingEnabled()) {
blk.extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(block.id);
if (blk.extras.feePercentiles !== null) {
blk.extras.medianFeeAmt = blk.extras.feePercentiles[3];
extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(block.id);
if (extras.feePercentiles !== null) {
extras.medianFeeAmt = extras.feePercentiles[3];
}
}
blk.extras.virtualSize = block.weight / 4.0;
if (blk.extras.coinbaseTx.vout.length > 0) {
blk.extras.coinbaseAddress = blk.extras.coinbaseTx.vout[0].scriptpubkey_address ?? null;
blk.extras.coinbaseSignature = blk.extras.coinbaseTx.vout[0].scriptpubkey_asm ?? null;
blk.extras.coinbaseSignatureAscii = transactionUtils.hex2ascii(blk.extras.coinbaseTx.vin[0].scriptsig) ?? null;
extras.virtualSize = block.weight / 4.0;
if (coinbaseTx?.vout.length > 0) {
extras.coinbaseAddress = coinbaseTx.vout[0].scriptpubkey_address ?? null;
extras.coinbaseSignature = coinbaseTx.vout[0].scriptpubkey_asm ?? null;
extras.coinbaseSignatureAscii = transactionUtils.hex2ascii(coinbaseTx.vin[0].scriptsig) ?? null;
} else {
blk.extras.coinbaseAddress = null;
blk.extras.coinbaseSignature = null;
blk.extras.coinbaseSignatureAscii = null;
extras.coinbaseAddress = null;
extras.coinbaseSignature = null;
extras.coinbaseSignatureAscii = null;
}
const header = await bitcoinClient.getBlockHeader(block.id, false);
blk.extras.header = header;
extras.header = header;
const coinStatsIndex = indexer.isCoreIndexReady('coinstatsindex');
if (coinStatsIndex !== null && coinStatsIndex.best_block_height >= block.height) {
const txoutset = await bitcoinClient.getTxoutSetinfo('none', block.height);
blk.extras.utxoSetSize = txoutset.txouts,
blk.extras.totalInputAmt = Math.round(txoutset.block_info.prevout_spent * 100000000);
extras.utxoSetSize = txoutset.txouts,
extras.totalInputAmt = Math.round(txoutset.block_info.prevout_spent * 100000000);
} else {
blk.extras.utxoSetSize = null;
blk.extras.totalInputAmt = null;
extras.utxoSetSize = null;
extras.totalInputAmt = null;
}
if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) {
let pool: PoolTag;
if (blk.extras?.coinbaseTx !== undefined) {
pool = await this.$findBlockMiner(blk.extras?.coinbaseTx);
if (coinbaseTx !== undefined) {
pool = await this.$findBlockMiner(coinbaseTx);
} else {
if (config.DATABASE.ENABLED === true) {
pool = await poolsRepository.$getUnknownPool();
@ -252,22 +252,24 @@ class Blocks {
logger.warn(`Cannot assign pool to block ${blk.height} and 'unknown' pool does not exist. ` +
`Check your "pools" table entries`);
} else {
blk.extras.pool = {
id: pool.id,
extras.pool = {
id: pool.uniqueId,
name: pool.name,
slug: pool.slug,
};
}
extras.matchRate = null;
if (config.MEMPOOL.AUDIT) {
const auditScore = await BlocksAuditsRepository.$getBlockAuditScore(block.id);
if (auditScore != null) {
blk.extras.matchRate = auditScore.matchRate;
extras.matchRate = auditScore.matchRate;
}
}
}
return blk;
blk.extras = <BlockExtension>extras;
return <BlockExtended>blk;
}
/**
@ -293,15 +295,18 @@ class Blocks {
} else {
pools = poolsParser.miningPools;
}
for (let i = 0; i < pools.length; ++i) {
if (address !== undefined) {
const addresses: string[] = JSON.parse(pools[i].addresses);
const addresses: string[] = typeof pools[i].addresses === 'string' ?
JSON.parse(pools[i].addresses) : pools[i].addresses;
if (addresses.indexOf(address) !== -1) {
return pools[i];
}
}
const regexes: string[] = JSON.parse(pools[i].regexes);
const regexes: string[] = typeof pools[i].regexes === 'string' ?
JSON.parse(pools[i].regexes) : pools[i].regexes;
for (let y = 0; y < regexes.length; ++y) {
const regex = new RegExp(regexes[y], 'i');
const match = asciiScriptSig.match(regex);
@ -652,7 +657,7 @@ class Blocks {
if (Common.indexingEnabled()) {
const dbBlock = await blocksRepository.$getBlockByHeight(height);
if (dbBlock !== null) {
return prepareBlock(dbBlock);
return dbBlock;
}
}
@ -665,11 +670,11 @@ class Blocks {
await blocksRepository.$saveBlockInDatabase(blockExtended);
}
return prepareBlock(blockExtended);
return blockExtended;
}
/**
* Index a block by hash if it's missing from the database. Returns the block after indexing
* Get one block by its hash
*/
public async $getBlock(hash: string): Promise<BlockExtended | IEsploraApi.Block> {
// Check the memory cache
@ -678,29 +683,14 @@ class Blocks {
return blockByHash;
}
// Block has already been indexed
if (Common.indexingEnabled()) {
const dbBlock = await blocksRepository.$getBlockByHash(hash);
if (dbBlock !== null) {
return prepareBlock(dbBlock);
}
}
// Not Bitcoin network, return the block as it
// Not Bitcoin network, return the block as it from the bitcoin backend
if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) {
return await bitcoinApi.$getBlock(hash);
}
// Bitcoin network, add our custom data on top
const block: IEsploraApi.Block = await bitcoinApi.$getBlock(hash);
const transactions = await this.$getTransactionsExtended(hash, block.height, true);
const blockExtended = await this.$getBlockExtended(block, transactions);
if (Common.indexingEnabled()) {
delete(blockExtended['coinbaseTx']);
await blocksRepository.$saveBlockInDatabase(blockExtended);
}
return blockExtended;
return await this.$indexBlock(block.height);
}
public async $getStrippedBlockTransactions(hash: string, skipMemoryCache = false,
@ -734,6 +724,18 @@ class Blocks {
return summary.transactions;
}
/**
* Get 15 blocks
*
* Internally this function uses two methods to get the blocks, and
* the method is automatically selected:
* - Using previous block hash links
* - Using block height
*
* @param fromHeight
* @param limit
* @returns
*/
public async $getBlocks(fromHeight?: number, limit: number = 15): Promise<BlockExtended[]> {
let currentHeight = fromHeight !== undefined ? fromHeight : this.currentBlockHeight;
if (currentHeight > this.currentBlockHeight) {
@ -759,11 +761,14 @@ class Blocks {
for (let i = 0; i < limit && currentHeight >= 0; i++) {
let block = this.getBlocks().find((b) => b.height === currentHeight);
if (block) {
// Using the memory cache (find by height)
returnBlocks.push(block);
} else if (Common.indexingEnabled()) {
// Using indexing (find by height, index on the fly, save in database)
block = await this.$indexBlock(currentHeight);
returnBlocks.push(block);
} else if (nextHash != null) {
} else if (nextHash !== null) {
// Without indexing, query block on the fly using bitoin backend, follow previous hash links
block = await this.$indexBlock(currentHeight);
nextHash = block.previousblockhash;
returnBlocks.push(block);
@ -788,7 +793,7 @@ class Blocks {
const blocks: any[] = [];
while (fromHeight <= toHeight) {
let block: any = await blocksRepository.$getBlockByHeight(fromHeight);
let block: BlockExtended | null = await blocksRepository.$getBlockByHeight(fromHeight);
if (!block) {
await this.$indexBlock(fromHeight);
block = await blocksRepository.$getBlockByHeight(fromHeight);
@ -801,11 +806,11 @@ class Blocks {
const cleanBlock: any = {
height: block.height ?? null,
hash: block.id ?? null,
timestamp: block.blockTimestamp ?? null,
median_timestamp: block.medianTime ?? null,
timestamp: block.timestamp ?? null,
median_timestamp: block.mediantime ?? null,
previous_block_hash: block.previousblockhash ?? null,
difficulty: block.difficulty ?? null,
header: block.header ?? null,
header: block.extras.header ?? null,
version: block.version ?? null,
bits: block.bits ?? null,
nonce: block.nonce ?? null,
@ -813,29 +818,30 @@ class Blocks {
weight: block.weight ?? null,
tx_count: block.tx_count ?? null,
merkle_root: block.merkle_root ?? null,
reward: block.reward ?? null,
total_fee_amt: block.fees ?? null,
avg_fee_amt: block.avg_fee ?? null,
median_fee_amt: block.median_fee_amt ?? null,
fee_amt_percentiles: block.fee_percentiles ?? null,
avg_fee_rate: block.avg_fee_rate ?? null,
median_fee_rate: block.median_fee ?? null,
fee_rate_percentiles: block.fee_span ?? null,
total_inputs: block.total_inputs ?? null,
total_input_amt: block.total_input_amt ?? null,
total_outputs: block.total_outputs ?? null,
total_output_amt: block.total_output_amt ?? null,
segwit_total_txs: block.segwit_total_txs ?? null,
segwit_total_size: block.segwit_total_size ?? null,
segwit_total_weight: block.segwit_total_weight ?? null,
avg_tx_size: block.avg_tx_size ?? null,
utxoset_change: block.utxoset_change ?? null,
utxoset_size: block.utxoset_size ?? null,
coinbase_raw: block.coinbase_raw ?? null,
coinbase_address: block.coinbase_address ?? null,
coinbase_signature: block.coinbase_signature ?? null,
coinbase_signature_ascii: block.coinbase_signature_ascii ?? null,
pool_slug: block.pool_slug ?? null,
reward: block.extras.reward ?? null,
total_fee_amt: block.extras.totalFees ?? null,
avg_fee_amt: block.extras.avgFee ?? null,
median_fee_amt: block.extras.medianFeeAmt ?? null,
fee_amt_percentiles: block.extras.feePercentiles ?? null,
avg_fee_rate: block.extras.avgFeeRate ?? null,
median_fee_rate: block.extras.medianFee ?? null,
fee_rate_percentiles: block.extras.feeRange ?? null,
total_inputs: block.extras.totalInputs ?? null,
total_input_amt: block.extras.totalInputAmt ?? null,
total_outputs: block.extras.totalOutputs ?? null,
total_output_amt: block.extras.totalOutputAmt ?? null,
segwit_total_txs: block.extras.segwitTotalTxs ?? null,
segwit_total_size: block.extras.segwitTotalSize ?? null,
segwit_total_weight: block.extras.segwitTotalWeight ?? null,
avg_tx_size: block.extras.avgTxSize ?? null,
utxoset_change: block.extras.utxoSetChange ?? null,
utxoset_size: block.extras.utxoSetSize ?? null,
coinbase_raw: block.extras.coinbaseRaw ?? null,
coinbase_address: block.extras.coinbaseAddress ?? null,
coinbase_signature: block.extras.coinbaseSignature ?? null,
coinbase_signature_ascii: block.extras.coinbaseSignatureAscii ?? null,
pool_slug: block.extras.pool.slug ?? null,
pool_id: block.extras.pool.id ?? null,
};
if (Common.blocksSummariesIndexingEnabled() && cleanBlock.fee_amt_percentiles === null) {

6
backend/src/api/chain-tips.ts
View File

@ -43,7 +43,11 @@ class ChainTips {
}
}
public getOrphanedBlocksAtHeight(height: number): OrphanedBlock[] {
public getOrphanedBlocksAtHeight(height: number | undefined): OrphanedBlock[] {
if (height === undefined) {
return [];
}
const orphans: OrphanedBlock[] = [];
for (const block of this.orphanedBlocks) {
if (block.height === height) {

2
backend/src/api/pools-parser.ts
View File

@ -7,6 +7,7 @@ import { PoolTag } from '../mempool.interfaces';
class PoolsParser {
miningPools: any[] = [];
unknownPool: any = {
'id': 0,
'name': 'Unknown',
'link': 'https://learnmeabitcoin.com/technical/coinbase-transaction',
'regexes': '[]',
@ -26,6 +27,7 @@ class PoolsParser {
public setMiningPools(pools): void {
for (const pool of pools) {
pool.regexes = pool.tags;
pool.slug = pool.name.replace(/[^a-z0-9]/gi, '').toLowerCase();
delete(pool.tags);
}
this.miningPools = pools;

9
backend/src/index.ts
View File

@ -179,7 +179,14 @@ class Server {
setTimeout(this.runMainUpdateLoop.bind(this), config.MEMPOOL.POLL_RATE_MS);
this.currentBackendRetryInterval = 5;
} catch (e: any) {
const loggerMsg = `runMainLoop error: ${(e instanceof Error ? e.message : e)}. Retrying in ${this.currentBackendRetryInterval} sec.`;
let loggerMsg = `Exception in runMainUpdateLoop(). Retrying in ${this.currentBackendRetryInterval} sec.`;
loggerMsg += ` Reason: ${(e instanceof Error ? e.message : e)}.`;
if (e?.stack) {
loggerMsg += ` Stack trace: ${e.stack}`;
}
// When we get a first Exception, only `logger.debug` it and retry after 5 seconds
// From the second Exception, `logger.warn` the Exception and increase the retry delay
// Maximum retry delay is 60 seconds
if (this.currentBackendRetryInterval > 5) {
logger.warn(loggerMsg);
mempool.setOutOfSync();

69
backend/src/mempool.interfaces.ts
View File

@ -1,9 +1,10 @@
import { IEsploraApi } from './api/bitcoin/esplora-api.interface';
import { OrphanedBlock } from './api/chain-tips';
import { HeapNode } from "./utils/pairing-heap";
import { HeapNode } from './utils/pairing-heap';
export interface PoolTag {
id: number; // mysql row id
id: number;
uniqueId: number;
name: string;
link: string;
regexes: string; // JSON array
@ -147,44 +148,44 @@ export interface TransactionStripped {
}
export interface BlockExtension {
totalFees?: number;
medianFee?: number;
feeRange?: number[];
reward?: number;
coinbaseTx?: TransactionMinerInfo;
matchRate?: number;
pool?: {
id: number;
totalFees: number;
medianFee: number; // median fee rate
feeRange: number[]; // fee rate percentiles
reward: number;
matchRate: number | null;
pool: {
id: number; // Note - This is the `unique_id`, not to mix with the auto increment `id`
name: string;
slug: string;
};
avgFee?: number;
avgFeeRate?: number;
coinbaseRaw?: string;
usd?: number | null;
medianTimestamp?: number;
blockTime?: number;
orphans?: OrphanedBlock[] | null;
coinbaseAddress?: string | null;
coinbaseSignature?: string | null;
coinbaseSignatureAscii?: string | null;
virtualSize?: number;
avgTxSize?: number;
totalInputs?: number;
totalOutputs?: number;
totalOutputAmt?: number;
medianFeeAmt?: number | null;
feePercentiles?: number[] | null,
segwitTotalTxs?: number;
segwitTotalSize?: number;
segwitTotalWeight?: number;
header?: string;
utxoSetChange?: number;
avgFee: number;
avgFeeRate: number;
coinbaseRaw: string;
orphans: OrphanedBlock[] | null;
coinbaseAddress: string | null;
coinbaseSignature: string | null;
coinbaseSignatureAscii: string | null;
virtualSize: number;
avgTxSize: number;
totalInputs: number;
totalOutputs: number;
totalOutputAmt: number;
medianFeeAmt: number | null; // median fee in sats
feePercentiles: number[] | null, // fee percentiles in sats
segwitTotalTxs: number;
segwitTotalSize: number;
segwitTotalWeight: number;
header: string;
utxoSetChange: number;
// Requires coinstatsindex, will be set to NULL otherwise
utxoSetSize?: number | null;
totalInputAmt?: number | null;
utxoSetSize: number | null;
totalInputAmt: number | null;
}
/**
* Note: Everything that is added in here will be automatically returned through
* /api/v1/block and /api/v1/blocks APIs
*/
export interface BlockExtended extends IEsploraApi.Block {
extras: BlockExtension;
}

206
backend/src/repositories/BlocksRepository.ts
View File

@ -1,8 +1,7 @@
import { BlockExtended, BlockPrice } from '../mempool.interfaces';
import { BlockExtended, BlockExtension, BlockPrice } from '../mempool.interfaces';
import DB from '../database';
import logger from '../logger';
import { Common } from '../api/common';
import { prepareBlock } from '../utils/blocks-utils';
import PoolsRepository from './PoolsRepository';
import HashratesRepository from './HashratesRepository';
import { escape } from 'mysql2';
@ -10,6 +9,50 @@ import BlocksSummariesRepository from './BlocksSummariesRepository';
import DifficultyAdjustmentsRepository from './DifficultyAdjustmentsRepository';
import bitcoinClient from '../api/bitcoin/bitcoin-client';
import config from '../config';
import chainTips from '../api/chain-tips';
import blocks from '../api/blocks';
const BLOCK_DB_FIELDS = `
blocks.hash AS id,
blocks.height,
blocks.version,
UNIX_TIMESTAMP(blocks.blockTimestamp) AS timestamp,
blocks.bits,
blocks.nonce,
blocks.difficulty,
blocks.merkle_root,
blocks.tx_count,
blocks.size,
blocks.weight,
blocks.previous_block_hash AS previousblockhash,
UNIX_TIMESTAMP(blocks.median_timestamp) AS mediantime,
blocks.fees AS totalFees,
blocks.median_fee AS medianFee,
blocks.fee_span AS feeRange,
blocks.reward,
pools.unique_id AS poolId,
pools.name AS poolName,
pools.slug AS poolSlug,
blocks.avg_fee AS avgFee,
blocks.avg_fee_rate AS avgFeeRate,
blocks.coinbase_raw AS coinbaseRaw,
blocks.coinbase_address AS coinbaseAddress,
blocks.coinbase_signature AS coinbaseSignature,
blocks.coinbase_signature_ascii AS coinbaseSignatureAscii,
blocks.avg_tx_size AS avgTxSize,
blocks.total_inputs AS totalInputs,
blocks.total_outputs AS totalOutputs,
blocks.total_output_amt AS totalOutputAmt,
blocks.median_fee_amt AS medianFeeAmt,
blocks.fee_percentiles AS feePercentiles,
blocks.segwit_total_txs AS segwitTotalTxs,
blocks.segwit_total_size AS segwitTotalSize,
blocks.segwit_total_weight AS segwitTotalWeight,
blocks.header,
blocks.utxoset_change AS utxoSetChange,
blocks.utxoset_size AS utxoSetSize,
blocks.total_input_amt AS totalInputAmts
`;
class BlocksRepository {
/**
@ -44,6 +87,11 @@ class BlocksRepository {
?, ?
)`;
const poolDbId = await PoolsRepository.$getPoolByUniqueId(block.extras.pool.id);
if (!poolDbId) {
throw Error(`Could not find a mining pool with the unique_id = ${block.extras.pool.id}. This error should never be printed.`);
}
const params: any[] = [
block.height,
block.id,
@ -53,7 +101,7 @@ class BlocksRepository {
block.tx_count,
block.extras.coinbaseRaw,
block.difficulty,
block.extras.pool?.id, // Should always be set to something
poolDbId.id,
block.extras.totalFees,
JSON.stringify(block.extras.feeRange),
block.extras.medianFee,
@ -65,7 +113,7 @@ class BlocksRepository {
block.previousblockhash,
block.extras.avgFee,
block.extras.avgFeeRate,
block.extras.medianTimestamp,
block.mediantime,
block.extras.header,
block.extras.coinbaseAddress,
truncatedCoinbaseSignature,
@ -87,9 +135,9 @@ class BlocksRepository {
await DB.query(query, params);
} catch (e: any) {
if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
logger.debug(`$saveBlockInDatabase() - Block ${block.height} has already been indexed, ignoring`);
logger.debug(`$saveBlockInDatabase() - Block ${block.height} has already been indexed, ignoring`, logger.tags.mining);
} else {
logger.err('Cannot save indexed block into db. Reason: ' + (e instanceof Error ? e.message : e));
logger.err('Cannot save indexed block into db. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
throw e;
}
}
@ -307,34 +355,17 @@ class BlocksRepository {
/**
* Get blocks mined by a specific mining pool
*/
public async $getBlocksByPool(slug: string, startHeight?: number): Promise<object[]> {
public async $getBlocksByPool(slug: string, startHeight?: number): Promise<BlockExtended[]> {
const pool = await PoolsRepository.$getPool(slug);
if (!pool) {
throw new Error('This mining pool does not exist ' + escape(slug));
}
const params: any[] = [];
let query = ` SELECT
blocks.height,
hash as id,
UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp,
size,
weight,
tx_count,
coinbase_raw,
difficulty,
fees,
fee_span,
median_fee,
reward,
version,
bits,
nonce,
merkle_root,
previous_block_hash as previousblockhash,
avg_fee,
avg_fee_rate
let query = `
SELECT ${BLOCK_DB_FIELDS}
FROM blocks
JOIN pools ON blocks.pool_id = pools.id
WHERE pool_id = ?`;
params.push(pool.id);
@ -347,11 +378,11 @@ class BlocksRepository {
LIMIT 10`;
try {
const [rows] = await DB.query(query, params);
const [rows]: any[] = await DB.query(query, params);
const blocks: BlockExtended[] = [];
for (const block of <object[]>rows) {
blocks.push(prepareBlock(block));
for (const block of rows) {
blocks.push(await this.formatDbBlockIntoExtendedBlock(block));
}
return blocks;
@ -364,32 +395,21 @@ class BlocksRepository {
/**
* Get one block by height
*/
public async $getBlockByHeight(height: number): Promise<object | null> {
public async $getBlockByHeight(height: number): Promise<BlockExtended | null> {
try {
const [rows]: any[] = await DB.query(`SELECT
blocks.*,
hash as id,
UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp,
UNIX_TIMESTAMP(blocks.median_timestamp) as medianTime,
pools.id as pool_id,
pools.name as pool_name,
pools.link as pool_link,
pools.slug as pool_slug,
pools.addresses as pool_addresses,
pools.regexes as pool_regexes,
previous_block_hash as previousblockhash
const [rows]: any[] = await DB.query(`
SELECT ${BLOCK_DB_FIELDS}
FROM blocks
JOIN pools ON blocks.pool_id = pools.id
WHERE blocks.height = ${height}
`);
WHERE blocks.height = ?`,
[height]
);
if (rows.length <= 0) {
return null;
}
rows[0].fee_span = JSON.parse(rows[0].fee_span);
rows[0].fee_percentiles = JSON.parse(rows[0].fee_percentiles);
return rows[0];
return await this.formatDbBlockIntoExtendedBlock(rows[0]);
} catch (e) {
logger.err(`Cannot get indexed block ${height}. Reason: ` + (e instanceof Error ? e.message : e));
throw e;
@ -402,10 +422,7 @@ class BlocksRepository {
public async $getBlockByHash(hash: string): Promise<object | null> {
try {
const query = `
SELECT *, blocks.height, UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp, hash as id,
pools.id as pool_id, pools.name as pool_name, pools.link as pool_link, pools.slug as pool_slug,
pools.addresses as pool_addresses, pools.regexes as pool_regexes,
previous_block_hash as previousblockhash
SELECT ${BLOCK_DB_FIELDS}
FROM blocks
JOIN pools ON blocks.pool_id = pools.id
WHERE hash = ?;
@ -415,9 +432,8 @@ class BlocksRepository {
if (rows.length <= 0) {
return null;
}
rows[0].fee_span = JSON.parse(rows[0].fee_span);
return rows[0];
return await this.formatDbBlockIntoExtendedBlock(rows[0]);
} catch (e) {
logger.err(`Cannot get indexed block ${hash}. Reason: ` + (e instanceof Error ? e.message : e));
throw e;
@ -833,6 +849,86 @@ class BlocksRepository {
throw e;
}
}
/**
* Convert a mysql row block into a BlockExtended. Note that you
* must provide the correct field into dbBlk object param
*
* @param dbBlk
*/
private async formatDbBlockIntoExtendedBlock(dbBlk: any): Promise<BlockExtended> {
const blk: Partial<BlockExtended> = {};
const extras: Partial<BlockExtension> = {};
// IEsploraApi.Block
blk.id = dbBlk.id;
blk.height = dbBlk.height;
blk.version = dbBlk.version;
blk.timestamp = dbBlk.timestamp;
blk.bits = dbBlk.bits;
blk.nonce = dbBlk.nonce;
blk.difficulty = dbBlk.difficulty;
blk.merkle_root = dbBlk.merkle_root;
blk.tx_count = dbBlk.tx_count;
blk.size = dbBlk.size;
blk.weight = dbBlk.weight;
blk.previousblockhash = dbBlk.previousblockhash;
blk.mediantime = dbBlk.mediantime;
// BlockExtension
extras.totalFees = dbBlk.totalFees;
extras.medianFee = dbBlk.medianFee;
extras.feeRange = JSON.parse(dbBlk.feeRange);
extras.reward = dbBlk.reward;
extras.pool = {
id: dbBlk.poolId,
name: dbBlk.poolName,
slug: dbBlk.poolSlug,
};
extras.avgFee = dbBlk.avgFee;
extras.avgFeeRate = dbBlk.avgFeeRate;
extras.coinbaseRaw = dbBlk.coinbaseRaw;
extras.coinbaseAddress = dbBlk.coinbaseAddress;
extras.coinbaseSignature = dbBlk.coinbaseSignature;
extras.coinbaseSignatureAscii = dbBlk.coinbaseSignatureAscii;
extras.avgTxSize = dbBlk.avgTxSize;
extras.totalInputs = dbBlk.totalInputs;
extras.totalOutputs = dbBlk.totalOutputs;
extras.totalOutputAmt = dbBlk.totalOutputAmt;
extras.medianFeeAmt = dbBlk.medianFeeAmt;
extras.feePercentiles = JSON.parse(dbBlk.feePercentiles);
extras.segwitTotalTxs = dbBlk.segwitTotalTxs;
extras.segwitTotalSize = dbBlk.segwitTotalSize;
extras.segwitTotalWeight = dbBlk.segwitTotalWeight;
extras.header = dbBlk.header,
extras.utxoSetChange = dbBlk.utxoSetChange;
extras.utxoSetSize = dbBlk.utxoSetSize;
extras.totalInputAmt = dbBlk.totalInputAmt;
extras.virtualSize = dbBlk.weight / 4.0;
// Re-org can happen after indexing so we need to always get the
// latest state from core
extras.orphans = chainTips.getOrphanedBlocksAtHeight(dbBlk.height);
// If we're missing block summary related field, check if we can populate them on the fly now
if (Common.blocksSummariesIndexingEnabled() &&
(extras.medianFeeAmt === null || extras.feePercentiles === null))
{
extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(dbBlk.id);
if (extras.feePercentiles === null) {
const block = await bitcoinClient.getBlock(dbBlk.id, 2);
const summary = blocks.summarizeBlock(block);
await BlocksSummariesRepository.$saveSummary({ height: block.height, mined: summary });
extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(dbBlk.id);
}
if (extras.feePercentiles !== null) {
extras.medianFeeAmt = extras.feePercentiles[3];
}
}
blk.extras = <BlockExtension>extras;
return <BlockExtended>blk;
}
}
export default new BlocksRepository();

6
backend/src/repositories/PoolsRepository.ts
View File

@ -10,7 +10,7 @@ class PoolsRepository {
* Get all pools tagging info
*/
public async $getPools(): Promise<PoolTag[]> {
const [rows] = await DB.query('SELECT id, name, addresses, regexes, slug FROM pools;');
const [rows] = await DB.query('SELECT id, unique_id as uniqueId, name, addresses, regexes, slug FROM pools');
return <PoolTag[]>rows;
}
@ -18,10 +18,10 @@ class PoolsRepository {
* Get unknown pool tagging info
*/
public async $getUnknownPool(): Promise<PoolTag> {
let [rows]: any[] = await DB.query('SELECT id, name, slug FROM pools where name = "Unknown"');
let [rows]: any[] = await DB.query('SELECT id, unique_id as uniqueId, name, slug FROM pools where name = "Unknown"');
if (rows && rows.length === 0 && config.DATABASE.ENABLED) {
await poolsParser.$insertUnknownPool();
[rows] = await DB.query('SELECT id, name, slug FROM pools where name = "Unknown"');
[rows] = await DB.query('SELECT id, unique_id as uniqueId, name, slug FROM pools where name = "Unknown"');
}
return <PoolTag>rows[0];
}

33
backend/src/utils/blocks-utils.ts
View File

@ -1,33 +0,0 @@
import { BlockExtended } from '../mempool.interfaces';
export function prepareBlock(block: any): BlockExtended {
return <BlockExtended>{
id: block.id ?? block.hash, // hash for indexed block
timestamp: block.timestamp ?? block.time ?? block.blockTimestamp, // blockTimestamp for indexed block
height: block.height,
version: block.version,
bits: (typeof block.bits === 'string' ? parseInt(block.bits, 16): block.bits),
nonce: block.nonce,
difficulty: block.difficulty,
merkle_root: block.merkle_root ?? block.merkleroot,
tx_count: block.tx_count ?? block.nTx,
size: block.size,
weight: block.weight,
previousblockhash: block.previousblockhash,
extras: {
coinbaseRaw: block.coinbase_raw ?? block.extras?.coinbaseRaw,
medianFee: block.medianFee ?? block.median_fee ?? block.extras?.medianFee,
feeRange: block.feeRange ?? block?.extras?.feeRange ?? block.fee_span,
reward: block.reward ?? block?.extras?.reward,
totalFees: block.totalFees ?? block?.fees ?? block?.extras?.totalFees,
avgFee: block?.extras?.avgFee ?? block.avg_fee,
avgFeeRate: block?.avgFeeRate ?? block.avg_fee_rate,
pool: block?.extras?.pool ?? (block?.pool_id ? {
id: block.pool_id,
name: block.pool_name,
slug: block.pool_slug,
} : undefined),
usd: block?.extras?.usd ?? block.usd ?? null,
}
};
}

1
frontend/src/app/interfaces/node-api.interface.ts
View File

@ -114,7 +114,6 @@ export interface BlockExtension {
medianFee?: number;
feeRange?: number[];
reward?: number;
coinbaseTx?: Transaction;
coinbaseRaw?: string;
matchRate?: number;
pool?: {