machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #3070 from mempool/nymkappa/feature/new-blocks-api

Browse Source 
Index coinstatsindex - Add bulk block query api
This commit is contained in:
wiz 2023-02-25 16:12:07 +09:00 committed by GitHub
commit a9c8971974
23 changed files with 601 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
backend/src/__fixtures__/mempool-config.template.json
View File

@ -3,7 +3,6 @@
"ENABLED": true,
"NETWORK": "__MEMPOOL_NETWORK__",
"BACKEND": "__MEMPOOL_BACKEND__",
"ENABLED": true,
"BLOCKS_SUMMARIES_INDEXING": true,
"HTTP_PORT": 1,
"SPAWN_CLUSTER_PROCS": 2,
@ -28,7 +27,8 @@
"AUDIT": "__MEMPOOL_AUDIT__",
"ADVANCED_GBT_AUDIT": "__MEMPOOL_ADVANCED_GBT_AUDIT__",
"ADVANCED_GBT_MEMPOOL": "__MEMPOOL_ADVANCED_GBT_MEMPOOL__",
"CPFP_INDEXING": "__MEMPOOL_CPFP_INDEXING__"
"CPFP_INDEXING": "__MEMPOOL_CPFP_INDEXING__",
"MAX_BLOCKS_BULK_QUERY": "__MEMPOOL_MAX_BLOCKS_BULK_QUERY__"
},
"CORE_RPC": {
"HOST": "__CORE_RPC_HOST__",

1
backend/src/__tests__/config.test.ts
View File

@ -41,6 +41,7 @@ describe('Mempool Backend Config', () => {
ADVANCED_GBT_AUDIT: false,
ADVANCED_GBT_MEMPOOL: false,
CPFP_INDEXING: false,
MAX_BLOCKS_BULK_QUERY: 0,
});
expect(config.ELECTRUM).toStrictEqual({ HOST: '127.0.0.1', PORT: 3306, TLS_ENABLED: true });

1
backend/src/api/bitcoin/bitcoin-api.ts
View File

@ -28,6 +28,7 @@ class BitcoinApi implements AbstractBitcoinApi {
size: block.size,
weight: block.weight,
previousblockhash: block.previousblockhash,
medianTime: block.mediantime,
};
}

37
backend/src/api/bitcoin/bitcoin.routes.ts
View File

@ -95,6 +95,8 @@ class BitcoinRoutes {
.get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/summary', this.getStrippedBlockTransactions)
.get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/audit-summary', this.getBlockAuditSummary)
.post(config.MEMPOOL.API_URL_PREFIX + 'psbt/addparents', this.postPsbtCompletion)
.get(config.MEMPOOL.API_URL_PREFIX + 'blocks-bulk/:from', this.getBlocksByBulk.bind(this))
.get(config.MEMPOOL.API_URL_PREFIX + 'blocks-bulk/:from/:to', this.getBlocksByBulk.bind(this))
;
if (config.MEMPOOL.BACKEND !== 'esplora') {
@ -402,6 +404,41 @@ class BitcoinRoutes {
}
}
private async getBlocksByBulk(req: Request, res: Response) {
try {
if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) { // Liquid, Bisq - Not implemented
return res.status(404).send(`This API is only available for Bitcoin networks`);
}
if (config.MEMPOOL.MAX_BLOCKS_BULK_QUERY <= 0) {
return res.status(404).send(`This API is disabled. Set config.MEMPOOL.MAX_BLOCKS_BULK_QUERY to a positive number to enable it.`);
}
if (!Common.indexingEnabled()) {
return res.status(404).send(`Indexing is required for this API`);
}
const from = parseInt(req.params.from, 10);
if (!req.params.from || from < 0) {
return res.status(400).send(`Parameter 'from' must be a block height (integer)`);
}
const to = req.params.to === undefined ? await bitcoinApi.$getBlockHeightTip() : parseInt(req.params.to, 10);
if (to < 0) {
return res.status(400).send(`Parameter 'to' must be a block height (integer)`);
}
if (from > to) {
return res.status(400).send(`Parameter 'to' must be a higher block height than 'from'`);
}
if ((to - from + 1) > config.MEMPOOL.MAX_BLOCKS_BULK_QUERY) {
return res.status(400).send(`You can only query ${config.MEMPOOL.MAX_BLOCKS_BULK_QUERY} blocks at once.`);
}
res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
res.json(await blocks.$getBlocksBetweenHeight(from, to));
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
}
}
private async getLegacyBlocks(req: Request, res: Response) {
try {
const returnBlocks: IEsploraApi.Block[] = [];

1
backend/src/api/bitcoin/esplora-api.interface.ts
View File

@ -88,6 +88,7 @@ export namespace IEsploraApi {
size: number;
weight: number;
previousblockhash: string;
medianTime?: number;
}
export interface Address {

205
backend/src/api/blocks.ts
View File

@ -25,6 +25,7 @@ import mining from './mining/mining';
import DifficultyAdjustmentsRepository from '../repositories/DifficultyAdjustmentsRepository';
import PricesRepository from '../repositories/PricesRepository';
import priceUpdater from '../tasks/price-updater';
import chainTips from './chain-tips';
class Blocks {
private blocks: BlockExtended[] = [];
@ -165,33 +166,80 @@ class Blocks {
* @returns BlockExtended
*/
private async $getBlockExtended(block: IEsploraApi.Block, transactions: TransactionExtended[]): Promise<BlockExtended> {
const blockExtended: BlockExtended = Object.assign({ extras: {} }, block);
blockExtended.extras.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0);
blockExtended.extras.coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]);
blockExtended.extras.coinbaseRaw = blockExtended.extras.coinbaseTx.vin[0].scriptsig;
blockExtended.extras.usd = priceUpdater.latestPrices.USD;
const blk: BlockExtended = Object.assign({ extras: {} }, block);
blk.extras.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0);
blk.extras.coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]);
blk.extras.coinbaseRaw = blk.extras.coinbaseTx.vin[0].scriptsig;
blk.extras.usd = priceUpdater.latestPrices.USD;
blk.extras.medianTimestamp = block.medianTime;
blk.extras.orphans = chainTips.getOrphanedBlocksAtHeight(blk.height);
if (block.height === 0) {
blockExtended.extras.medianFee = 0; // 50th percentiles
blockExtended.extras.feeRange = [0, 0, 0, 0, 0, 0, 0];
blockExtended.extras.totalFees = 0;
blockExtended.extras.avgFee = 0;
blockExtended.extras.avgFeeRate = 0;
blk.extras.medianFee = 0; // 50th percentiles
blk.extras.feeRange = [0, 0, 0, 0, 0, 0, 0];
blk.extras.totalFees = 0;
blk.extras.avgFee = 0;
blk.extras.avgFeeRate = 0;
blk.extras.utxoSetChange = 0;
blk.extras.avgTxSize = 0;
blk.extras.totalInputs = 0;
blk.extras.totalOutputs = 1;
blk.extras.totalOutputAmt = 0;
blk.extras.segwitTotalTxs = 0;
blk.extras.segwitTotalSize = 0;
blk.extras.segwitTotalWeight = 0;
} else {
const stats = await bitcoinClient.getBlockStats(block.id, [
'feerate_percentiles', 'minfeerate', 'maxfeerate', 'totalfee', 'avgfee', 'avgfeerate'
]);
blockExtended.extras.medianFee = stats.feerate_percentiles[2]; // 50th percentiles
blockExtended.extras.feeRange = [stats.minfeerate, stats.feerate_percentiles, stats.maxfeerate].flat();
blockExtended.extras.totalFees = stats.totalfee;
blockExtended.extras.avgFee = stats.avgfee;
blockExtended.extras.avgFeeRate = stats.avgfeerate;
const stats = await bitcoinClient.getBlockStats(block.id);
blk.extras.medianFee = stats.feerate_percentiles[2]; // 50th percentiles
blk.extras.feeRange = [stats.minfeerate, stats.feerate_percentiles, stats.maxfeerate].flat();
blk.extras.totalFees = stats.totalfee;
blk.extras.avgFee = stats.avgfee;
blk.extras.avgFeeRate = stats.avgfeerate;
blk.extras.utxoSetChange = stats.utxo_increase;
blk.extras.avgTxSize = Math.round(stats.total_size / stats.txs * 100) * 0.01;
blk.extras.totalInputs = stats.ins;
blk.extras.totalOutputs = stats.outs;
blk.extras.totalOutputAmt = stats.total_out;
blk.extras.segwitTotalTxs = stats.swtxs;
blk.extras.segwitTotalSize = stats.swtotal_size;
blk.extras.segwitTotalWeight = stats.swtotal_weight;
}
if (Common.blocksSummariesIndexingEnabled()) {
blk.extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(block.id);
if (blk.extras.feePercentiles !== null) {
blk.extras.medianFeeAmt = blk.extras.feePercentiles[3];
}
}
blk.extras.virtualSize = block.weight / 4.0;
if (blk.extras.coinbaseTx.vout.length > 0) {
blk.extras.coinbaseAddress = blk.extras.coinbaseTx.vout[0].scriptpubkey_address ?? null;
blk.extras.coinbaseSignature = blk.extras.coinbaseTx.vout[0].scriptpubkey_asm ?? null;
blk.extras.coinbaseSignatureAscii = transactionUtils.hex2ascii(blk.extras.coinbaseTx.vin[0].scriptsig) ?? null;
} else {
blk.extras.coinbaseAddress = null;
blk.extras.coinbaseSignature = null;
blk.extras.coinbaseSignatureAscii = null;
}
const header = await bitcoinClient.getBlockHeader(block.id, false);
blk.extras.header = header;
const coinStatsIndex = indexer.isCoreIndexReady('coinstatsindex');
if (coinStatsIndex !== null && coinStatsIndex.best_block_height >= block.height) {
const txoutset = await bitcoinClient.getTxoutSetinfo('none', block.height);
blk.extras.utxoSetSize = txoutset.txouts,
blk.extras.totalInputAmt = Math.round(txoutset.block_info.prevout_spent * 100000000);
} else {
blk.extras.utxoSetSize = null;
blk.extras.totalInputAmt = null;
}
if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) {
let pool: PoolTag;
if (blockExtended.extras?.coinbaseTx !== undefined) {
pool = await this.$findBlockMiner(blockExtended.extras?.coinbaseTx);
if (blk.extras?.coinbaseTx !== undefined) {
pool = await this.$findBlockMiner(blk.extras?.coinbaseTx);
} else {
if (config.DATABASE.ENABLED === true) {
pool = await poolsRepository.$getUnknownPool();
@ -201,10 +249,10 @@ class Blocks {
}
if (!pool) { // We should never have this situation in practise
logger.warn(`Cannot assign pool to block ${blockExtended.height} and 'unknown' pool does not exist. ` +
logger.warn(`Cannot assign pool to block ${blk.height} and 'unknown' pool does not exist. ` +
`Check your "pools" table entries`);
} else {
blockExtended.extras.pool = {
blk.extras.pool = {
id: pool.id,
name: pool.name,
slug: pool.slug,
@ -214,12 +262,12 @@ class Blocks {
if (config.MEMPOOL.AUDIT) {
const auditScore = await BlocksAuditsRepository.$getBlockAuditScore(block.id);
if (auditScore != null) {
blockExtended.extras.matchRate = auditScore.matchRate;
blk.extras.matchRate = auditScore.matchRate;
}
}
}
return blockExtended;
return blk;
}
/**
@ -500,6 +548,7 @@ class Blocks {
} else {
this.currentBlockHeight++;
logger.debug(`New block found (#${this.currentBlockHeight})!`);
await chainTips.updateOrphanedBlocks();
}
const blockHash = await bitcoinApi.$getBlockHash(this.currentBlockHeight);
@ -688,7 +737,6 @@ class Blocks {
}
public async $getBlocks(fromHeight?: number, limit: number = 15): Promise<BlockExtended[]> {
let currentHeight = fromHeight !== undefined ? fromHeight : this.currentBlockHeight;
if (currentHeight > this.currentBlockHeight) {
limit -= currentHeight - this.currentBlockHeight;
@ -728,6 +776,113 @@ class Blocks {
return returnBlocks;
}
/**
* Used for bulk block data query
*
* @param fromHeight
* @param toHeight
*/
public async $getBlocksBetweenHeight(fromHeight: number, toHeight: number): Promise<any> {
if (!Common.indexingEnabled()) {
return [];
}
const blocks: any[] = [];
while (fromHeight <= toHeight) {
let block: any = await blocksRepository.$getBlockByHeight(fromHeight);
if (!block) {
await this.$indexBlock(fromHeight);
block = await blocksRepository.$getBlockByHeight(fromHeight);
if (!block) {
continue;
}
}
// Cleanup fields before sending the response
const cleanBlock: any = {
height: block.height ?? null,
hash: block.id ?? null,
timestamp: block.blockTimestamp ?? null,
median_timestamp: block.medianTime ?? null,
previous_block_hash: block.previousblockhash ?? null,
difficulty: block.difficulty ?? null,
header: block.header ?? null,
version: block.version ?? null,
bits: block.bits ?? null,
nonce: block.nonce ?? null,
size: block.size ?? null,
weight: block.weight ?? null,
tx_count: block.tx_count ?? null,
merkle_root: block.merkle_root ?? null,
reward: block.reward ?? null,
total_fee_amt: block.fees ?? null,
avg_fee_amt: block.avg_fee ?? null,
median_fee_amt: block.median_fee_amt ?? null,
fee_amt_percentiles: block.fee_percentiles ?? null,
avg_fee_rate: block.avg_fee_rate ?? null,
median_fee_rate: block.median_fee ?? null,
fee_rate_percentiles: block.fee_span ?? null,
total_inputs: block.total_inputs ?? null,
total_input_amt: block.total_input_amt ?? null,
total_outputs: block.total_outputs ?? null,
total_output_amt: block.total_output_amt ?? null,
segwit_total_txs: block.segwit_total_txs ?? null,
segwit_total_size: block.segwit_total_size ?? null,
segwit_total_weight: block.segwit_total_weight ?? null,
avg_tx_size: block.avg_tx_size ?? null,
utxoset_change: block.utxoset_change ?? null,
utxoset_size: block.utxoset_size ?? null,
coinbase_raw: block.coinbase_raw ?? null,
coinbase_address: block.coinbase_address ?? null,
coinbase_signature: block.coinbase_signature ?? null,
coinbase_signature_ascii: block.coinbase_signature_ascii ?? null,
pool_slug: block.pool_slug ?? null,
};
if (Common.blocksSummariesIndexingEnabled() && cleanBlock.fee_amt_percentiles === null) {
cleanBlock.fee_amt_percentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(cleanBlock.hash);
if (cleanBlock.fee_amt_percentiles === null) {
const block = await bitcoinClient.getBlock(cleanBlock.hash, 2);
const summary = this.summarizeBlock(block);
await BlocksSummariesRepository.$saveSummary({ height: block.height, mined: summary });
cleanBlock.fee_amt_percentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(cleanBlock.hash);
}
if (cleanBlock.fee_amt_percentiles !== null) {
cleanBlock.median_fee_amt = cleanBlock.fee_amt_percentiles[3];
}
}
cleanBlock.fee_amt_percentiles = {
'min': cleanBlock.fee_amt_percentiles[0],
'perc_10': cleanBlock.fee_amt_percentiles[1],
'perc_25': cleanBlock.fee_amt_percentiles[2],
'perc_50': cleanBlock.fee_amt_percentiles[3],
'perc_75': cleanBlock.fee_amt_percentiles[4],
'perc_90': cleanBlock.fee_amt_percentiles[5],
'max': cleanBlock.fee_amt_percentiles[6],
};
cleanBlock.fee_rate_percentiles = {
'min': cleanBlock.fee_rate_percentiles[0],
'perc_10': cleanBlock.fee_rate_percentiles[1],
'perc_25': cleanBlock.fee_rate_percentiles[2],
'perc_50': cleanBlock.fee_rate_percentiles[3],
'perc_75': cleanBlock.fee_rate_percentiles[4],
'perc_90': cleanBlock.fee_rate_percentiles[5],
'max': cleanBlock.fee_rate_percentiles[6],
};
// Re-org can happen after indexing so we need to always get the
// latest state from core
cleanBlock.orphans = chainTips.getOrphanedBlocksAtHeight(cleanBlock.height);
blocks.push(cleanBlock);
fromHeight++;
}
return blocks;
}
public async $getBlockAuditSummary(hash: string): Promise<any> {
let summary;
if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) {

57
backend/src/api/chain-tips.ts Normal file
View File

@ -0,0 +1,57 @@
import logger from "../logger";
import bitcoinClient from "./bitcoin/bitcoin-client";
export interface ChainTip {
height: number;
hash: string;
branchlen: number;
status: 'invalid' | 'active' | 'valid-fork' | 'valid-headers' | 'headers-only';
};
export interface OrphanedBlock {
height: number;
hash: string;
status: 'valid-fork' | 'valid-headers' | 'headers-only';
}
class ChainTips {
private chainTips: ChainTip[] = [];
private orphanedBlocks: OrphanedBlock[] = [];
public async updateOrphanedBlocks(): Promise<void> {
try {
this.chainTips = await bitcoinClient.getChainTips();
this.orphanedBlocks = [];
for (const chain of this.chainTips) {
if (chain.status === 'valid-fork' || chain.status === 'valid-headers') {
let block = await bitcoinClient.getBlock(chain.hash);
while (block && block.confirmations === -1) {
this.orphanedBlocks.push({
height: block.height,
hash: block.hash,
status: chain.status
});
block = await bitcoinClient.getBlock(block.previousblockhash);
}
}
}
logger.debug(`Updated orphaned blocks cache. Found ${this.orphanedBlocks.length} orphaned blocks`);
} catch (e) {
logger.err(`Cannot get fetch orphaned blocks. Reason: ${e instanceof Error ? e.message : e}`);
}
}
public getOrphanedBlocksAtHeight(height: number): OrphanedBlock[] {
const orphans: OrphanedBlock[] = [];
for (const block of this.orphanedBlocks) {
if (block.height === height) {
orphans.push(block);
}
}
return orphans;
}
}
export default new ChainTips();

31
backend/src/api/database-migration.ts
View File

@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository';
import { RowDataPacket } from 'mysql2';
class DatabaseMigration {
private static currentVersion = 54;
private static currentVersion = 55;
private queryTimeout = 3600_000;
private statisticsAddedIndexed = false;
private uniqueLogs: string[] = [];
@ -483,6 +483,13 @@ class DatabaseMigration {
}
await this.updateToSchemaVersion(54);
}
if (databaseSchemaVersion < 55) {
await this.$executeQuery(this.getAdditionalBlocksDataQuery());
this.uniqueLog(logger.notice, this.blocksTruncatedMessage);
await this.$executeQuery('TRUNCATE blocks;'); // Need to re-index
await this.updateToSchemaVersion(55);
}
}
/**
@ -756,6 +763,28 @@ class DatabaseMigration {
) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
}
private getAdditionalBlocksDataQuery(): string {
return `ALTER TABLE blocks
ADD median_timestamp timestamp NOT NULL,
ADD coinbase_address varchar(100) NULL,
ADD coinbase_signature varchar(500) NULL,
ADD coinbase_signature_ascii varchar(500) NULL,
ADD avg_tx_size double unsigned NOT NULL,
ADD total_inputs int unsigned NOT NULL,
ADD total_outputs int unsigned NOT NULL,
ADD total_output_amt bigint unsigned NOT NULL,
ADD fee_percentiles longtext NULL,
ADD median_fee_amt int unsigned NULL,
ADD segwit_total_txs int unsigned NOT NULL,
ADD segwit_total_size int unsigned NOT NULL,
ADD segwit_total_weight int unsigned NOT NULL,
ADD header varchar(160) NOT NULL,
ADD utxoset_change int NOT NULL,
ADD utxoset_size int unsigned NULL,
ADD total_input_amt bigint unsigned NULL
`;
}
private getCreateDailyStatsTableQuery(): string {
return `CREATE TABLE IF NOT EXISTS hashrates (
hashrate_timestamp timestamp NOT NULL,

2
backend/src/api/disk-cache.ts
View File

@ -9,7 +9,7 @@ import { TransactionExtended } from '../mempool.interfaces';
import { Common } from './common';
class DiskCache {
private cacheSchemaVersion = 1;
private cacheSchemaVersion = 2;
private static FILE_NAME = config.MEMPOOL.CACHE_DIR + '/cache.json';
private static FILE_NAMES = config.MEMPOOL.CACHE_DIR + '/cache{number}.json';

41
backend/src/api/mining/mining.ts
View File

@ -172,7 +172,7 @@ class Mining {
}
/**
* [INDEXING] Generate weekly mining pool hashrate history
* Generate weekly mining pool hashrate history
*/
public async $generatePoolHashrateHistory(): Promise<void> {
const now = new Date();
@ -279,7 +279,7 @@ class Mining {
}
/**
* [INDEXING] Generate daily hashrate data
* Generate daily hashrate data
*/
public async $generateNetworkHashrateHistory(): Promise<void> {
// We only run this once a day around midnight
@ -459,7 +459,7 @@ class Mining {
/**
* Create a link between blocks and the latest price at when they were mined
*/
public async $indexBlockPrices() {
public async $indexBlockPrices(): Promise<void> {
if (this.blocksPriceIndexingRunning === true) {
return;
}
@ -520,6 +520,41 @@ class Mining {
this.blocksPriceIndexingRunning = false;
}
/**
* Index core coinstatsindex
*/
public async $indexCoinStatsIndex(): Promise<void> {
let timer = new Date().getTime() / 1000;
let totalIndexed = 0;
const blockchainInfo = await bitcoinClient.getBlockchainInfo();
let currentBlockHeight = blockchainInfo.blocks;
while (currentBlockHeight > 0) {
const indexedBlocks = await BlocksRepository.$getBlocksMissingCoinStatsIndex(
currentBlockHeight, currentBlockHeight - 10000);
for (const block of indexedBlocks) {
const txoutset = await bitcoinClient.getTxoutSetinfo('none', block.height);
await BlocksRepository.$updateCoinStatsIndexData(block.hash, txoutset.txouts,
Math.round(txoutset.block_info.prevout_spent * 100000000));
++totalIndexed;
const elapsedSeconds = Math.max(1, new Date().getTime() / 1000 - timer);
if (elapsedSeconds > 5) {
logger.info(`Indexing coinstatsindex data for block #${block.height}. Indexed ${totalIndexed} blocks.`, logger.tags.mining);
timer = new Date().getTime() / 1000;
}
}
currentBlockHeight -= 10000;
}
if (totalIndexed) {
logger.info(`Indexing missing coinstatsindex data completed`, logger.tags.mining);
}
}
private getDateMidnight(date: Date): Date {
date.setUTCHours(0);
date.setUTCMinutes(0);

1
backend/src/api/transaction-utils.ts
View File

@ -14,6 +14,7 @@ class TransactionUtils {
vout: tx.vout
.map((vout) => ({
scriptpubkey_address: vout.scriptpubkey_address,
scriptpubkey_asm: vout.scriptpubkey_asm,
value: vout.value
}))
.filter((vout) => vout.value)

2
backend/src/config.ts
View File

@ -32,6 +32,7 @@ interface IConfig {
ADVANCED_GBT_AUDIT: boolean;
ADVANCED_GBT_MEMPOOL: boolean;
CPFP_INDEXING: boolean;
MAX_BLOCKS_BULK_QUERY: number;
};
ESPLORA: {
REST_API_URL: string;
@ -153,6 +154,7 @@ const defaults: IConfig = {
'ADVANCED_GBT_AUDIT': false,
'ADVANCED_GBT_MEMPOOL': false,
'CPFP_INDEXING': false,
'MAX_BLOCKS_BULK_QUERY': 0,
},
'ESPLORA': {
'REST_API_URL': 'http://127.0.0.1:3000',

3
backend/src/database.ts
View File

@ -24,7 +24,8 @@ import { FieldPacket, OkPacket, PoolOptions, ResultSetHeader, RowDataPacket } fr
private checkDBFlag() {
if (config.DATABASE.ENABLED === false) {
logger.err('Trying to use DB feature but config.DATABASE.ENABLED is set to false, please open an issue');
const stack = new Error().stack;
logger.err(`Trying to use DB feature but config.DATABASE.ENABLED is set to false, please open an issue.\nStack trace: ${stack}}`);
}
}

3
backend/src/index.ts
View File

@ -36,6 +36,8 @@ import bitcoinRoutes from './api/bitcoin/bitcoin.routes';
import fundingTxFetcher from './tasks/lightning/sync-tasks/funding-tx-fetcher';
import forensicsService from './tasks/lightning/forensics.service';
import priceUpdater from './tasks/price-updater';
import mining from './api/mining/mining';
import chainTips from './api/chain-tips';
import { AxiosError } from 'axios';
class Server {
@ -133,6 +135,7 @@ class Server {
}
priceUpdater.$run();
await chainTips.updateOrphanedBlocks();
this.setUpHttpApiRoutes();

76
backend/src/indexer.ts
View File

@ -8,18 +8,67 @@ import bitcoinClient from './api/bitcoin/bitcoin-client';
import priceUpdater from './tasks/price-updater';
import PricesRepository from './repositories/PricesRepository';
export interface CoreIndex {
name: string;
synced: boolean;
best_block_height: number;
}
class Indexer {
runIndexer = true;
indexerRunning = false;
tasksRunning: string[] = [];
coreIndexes: CoreIndex[] = [];
public reindex() {
/**
* Check which core index is available for indexing
*/
public async checkAvailableCoreIndexes(): Promise<void> {
const updatedCoreIndexes: CoreIndex[] = [];
const indexes: any = await bitcoinClient.getIndexInfo();
for (const indexName in indexes) {
const newState = {
name: indexName,
synced: indexes[indexName].synced,
best_block_height: indexes[indexName].best_block_height,
};
logger.info(`Core index '${indexName}' is ${indexes[indexName].synced ? 'synced' : 'not synced'}. Best block height is ${indexes[indexName].best_block_height}`);
updatedCoreIndexes.push(newState);
if (indexName === 'coinstatsindex' && newState.synced === true) {
const previousState = this.isCoreIndexReady('coinstatsindex');
// if (!previousState || previousState.synced === false) {
this.runSingleTask('coinStatsIndex');
// }
}
}
this.coreIndexes = updatedCoreIndexes;
}
/**
* Return the best block height if a core index is available, or 0 if not
*
* @param name
* @returns
*/
public isCoreIndexReady(name: string): CoreIndex | null {
for (const index of this.coreIndexes) {
if (index.name === name && index.synced === true) {
return index;
}
}
return null;
}
public reindex(): void {
if (Common.indexingEnabled()) {
this.runIndexer = true;
}
}
public async runSingleTask(task: 'blocksPrices') {
public async runSingleTask(task: 'blocksPrices' | 'coinStatsIndex'): Promise<void> {
if (!Common.indexingEnabled()) {
return;
}
@ -28,20 +77,27 @@ class Indexer {
this.tasksRunning.push(task);
const lastestPriceId = await PricesRepository.$getLatestPriceId();
if (priceUpdater.historyInserted === false || lastestPriceId === null) {
logger.debug(`Blocks prices indexer is waiting for the price updater to complete`)
logger.debug(`Blocks prices indexer is waiting for the price updater to complete`);
setTimeout(() => {
this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask != task)
this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask !== task);
this.runSingleTask('blocksPrices');
}, 10000);
} else {
logger.debug(`Blocks prices indexer will run now`)
logger.debug(`Blocks prices indexer will run now`);
await mining.$indexBlockPrices();
this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask != task)
this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask !== task);
}
}
if (task === 'coinStatsIndex' && !this.tasksRunning.includes(task)) {
this.tasksRunning.push(task);
logger.debug(`Indexing coinStatsIndex now`);
await mining.$indexCoinStatsIndex();
this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask !== task);
}
}
public async $run() {
public async $run(): Promise<void> {
if (!Common.indexingEnabled() || this.runIndexer === false ||
this.indexerRunning === true || mempool.hasPriority()
) {
@ -57,7 +113,9 @@ class Indexer {
this.runIndexer = false;
this.indexerRunning = true;
logger.debug(`Running mining indexer`);
logger.info(`Running mining indexer`);
await this.checkAvailableCoreIndexes();
try {
await priceUpdater.$run();
@ -93,7 +151,7 @@ class Indexer {
setTimeout(() => this.reindex(), runEvery);
}
async $resetHashratesIndexingState() {
async $resetHashratesIndexingState(): Promise<void> {
try {
await HashratesRepository.$setLatestRun('last_hashrates_indexing', 0);
await HashratesRepository.$setLatestRun('last_weekly_hashrates_indexing', 0);

23
backend/src/mempool.interfaces.ts
View File

@ -1,4 +1,5 @@
import { IEsploraApi } from './api/bitcoin/esplora-api.interface';
import { OrphanedBlock } from './api/chain-tips';
import { HeapNode } from "./utils/pairing-heap";
export interface PoolTag {
@ -64,6 +65,7 @@ interface VinStrippedToScriptsig {
interface VoutStrippedToScriptPubkey {
scriptpubkey_address: string | undefined;
scriptpubkey_asm: string | undefined;
value: number;
}
@ -160,6 +162,27 @@ export interface BlockExtension {
avgFeeRate?: number;
coinbaseRaw?: string;
usd?: number | null;
medianTimestamp?: number;
blockTime?: number;
orphans?: OrphanedBlock[] | null;
coinbaseAddress?: string | null;
coinbaseSignature?: string | null;
coinbaseSignatureAscii?: string | null;
virtualSize?: number;
avgTxSize?: number;
totalInputs?: number;
totalOutputs?: number;
totalOutputAmt?: number;
medianFeeAmt?: number | null;
feePercentiles?: number[] | null,
segwitTotalTxs?: number;
segwitTotalSize?: number;
segwitTotalWeight?: number;
header?: string;
utxoSetChange?: number;
// Requires coinstatsindex, will be set to NULL otherwise
utxoSetSize?: number | null;
totalInputAmt?: number | null;
}
export interface BlockExtended extends IEsploraApi.Block {

128
backend/src/repositories/BlocksRepository.ts
View File

@ -18,17 +18,27 @@ class BlocksRepository {
public async $saveBlockInDatabase(block: BlockExtended) {
try {
const query = `INSERT INTO blocks(
height, hash, blockTimestamp, size,
weight, tx_count, coinbase_raw, difficulty,
pool_id, fees, fee_span, median_fee,
reward, version, bits, nonce,
merkle_root, previous_block_hash, avg_fee, avg_fee_rate
height, hash, blockTimestamp, size,
weight, tx_count, coinbase_raw, difficulty,
pool_id, fees, fee_span, median_fee,
reward, version, bits, nonce,
merkle_root, previous_block_hash, avg_fee, avg_fee_rate,
median_timestamp, header, coinbase_address,
coinbase_signature, utxoset_size, utxoset_change, avg_tx_size,
total_inputs, total_outputs, total_input_amt, total_output_amt,
fee_percentiles, segwit_total_txs, segwit_total_size, segwit_total_weight,
median_fee_amt, coinbase_signature_ascii
) VALUE (
?, ?, FROM_UNIXTIME(?), ?,
?, ?, ?, ?,
?, ?, ?, ?,
?, ?, ?, ?,
?, ?, ?, ?
?, ?, ?, ?,
FROM_UNIXTIME(?), ?, ?,
?, ?, ?, ?,
?, ?, ?, ?,
?, ?, ?, ?,
?, ?
)`;
const params: any[] = [
@ -52,6 +62,23 @@ class BlocksRepository {
block.previousblockhash,
block.extras.avgFee,
block.extras.avgFeeRate,
block.extras.medianTimestamp,
block.extras.header,
block.extras.coinbaseAddress,
block.extras.coinbaseSignature,
block.extras.utxoSetSize,
block.extras.utxoSetChange,
block.extras.avgTxSize,
block.extras.totalInputs,
block.extras.totalOutputs,
block.extras.totalInputAmt,
block.extras.totalOutputAmt,
block.extras.feePercentiles ? JSON.stringify(block.extras.feePercentiles) : null,
block.extras.segwitTotalTxs,
block.extras.segwitTotalSize,
block.extras.segwitTotalWeight,
block.extras.medianFeeAmt,
block.extras.coinbaseSignatureAscii,
];
await DB.query(query, params);
@ -65,6 +92,33 @@ class BlocksRepository {
}
}
/**
* Save newly indexed data from core coinstatsindex
*
* @param utxoSetSize
* @param totalInputAmt
*/
public async $updateCoinStatsIndexData(blockHash: string, utxoSetSize: number,
totalInputAmt: number
) : Promise<void> {
try {
const query = `
UPDATE blocks
SET utxoset_size = ?, total_input_amt = ?
WHERE hash = ?
`;
const params: any[] = [
utxoSetSize,
totalInputAmt,
blockHash
];
await DB.query(query, params);
} catch (e: any) {
logger.err('Cannot update indexed block coinstatsindex. Reason: ' + (e instanceof Error ? e.message : e));
throw e;
}
}
/**
* Get all block height that have not been indexed between [startHeight, endHeight]
*/
@ -310,32 +364,17 @@ class BlocksRepository {
public async $getBlockByHeight(height: number): Promise<object | null> {
try {
const [rows]: any[] = await DB.query(`SELECT
blocks.height,
hash,
blocks.*,
hash as id,
UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp,
size,
weight,
tx_count,
coinbase_raw,
difficulty,
UNIX_TIMESTAMP(blocks.median_timestamp) as medianTime,
pools.id as pool_id,
pools.name as pool_name,
pools.link as pool_link,
pools.slug as pool_slug,
pools.addresses as pool_addresses,
pools.regexes as pool_regexes,
fees,
fee_span,
median_fee,
reward,
version,
bits,
nonce,
merkle_root,
previous_block_hash as previousblockhash,
avg_fee,
avg_fee_rate
previous_block_hash as previousblockhash
FROM blocks
JOIN pools ON blocks.pool_id = pools.id
WHERE blocks.height = ${height}
@ -346,6 +385,7 @@ class BlocksRepository {
}
rows[0].fee_span = JSON.parse(rows[0].fee_span);
rows[0].fee_percentiles = JSON.parse(rows[0].fee_percentiles);
return rows[0];
} catch (e) {
logger.err(`Cannot get indexed block ${height}. Reason: ` + (e instanceof Error ? e.message : e));
@ -694,7 +734,6 @@ class BlocksRepository {
logger.err('Cannot fetch CPFP unindexed blocks. Reason: ' + (e instanceof Error ? e.message : e));
throw e;
}
return [];
}
/**
@ -741,7 +780,7 @@ class BlocksRepository {
try {
let query = `INSERT INTO blocks_prices(height, price_id) VALUES`;
for (const price of blockPrices) {
query += ` (${price.height}, ${price.priceId}),`
query += ` (${price.height}, ${price.priceId}),`;
}
query = query.slice(0, -1);
await DB.query(query);
@ -754,6 +793,43 @@ class BlocksRepository {
}
}
}
/**
* Get all indexed blocsk with missing coinstatsindex data
*/
public async $getBlocksMissingCoinStatsIndex(maxHeight: number, minHeight: number): Promise<any> {
try {
const [blocks] = await DB.query(`
SELECT height, hash
FROM blocks
WHERE height >= ${minHeight} AND height <= ${maxHeight} AND
(utxoset_size IS NULL OR total_input_amt IS NULL)
`);
return blocks;
} catch (e) {
logger.err(`Cannot get blocks with missing coinstatsindex. Reason: ` + (e instanceof Error ? e.message : e));
throw e;
}
}
/**
* Save indexed median fee to avoid recomputing it later
*
* @param id
* @param feePercentiles
*/
public async $saveFeePercentilesForBlockId(id: string, feePercentiles: number[]): Promise<void> {
try {
await DB.query(`
UPDATE blocks SET fee_percentiles = ?, median_fee_amt = ?
WHERE hash = ?`,
[JSON.stringify(feePercentiles), feePercentiles[3], id]
);
} catch (e) {
logger.err(`Cannot update block fee_percentiles. Reason: ` + (e instanceof Error ? e.message : e));
throw e;
}
}
}
export default new BlocksRepository();

42
backend/src/repositories/BlocksSummariesRepository.ts
View File

@ -80,6 +80,48 @@ class BlocksSummariesRepository {
logger.err('Cannot delete indexed blocks summaries. Reason: ' + (e instanceof Error ? e.message : e));
}
}
/**
* Get the fee percentiles if the block has already been indexed, [] otherwise
*
* @param id
*/
public async $getFeePercentilesByBlockId(id: string): Promise<number[] | null> {
try {
const [rows]: any[] = await DB.query(`
SELECT transactions
FROM blocks_summaries
WHERE id = ?`,
[id]
);
if (rows === null || rows.length === 0) {
return null;
}
const transactions = JSON.parse(rows[0].transactions);
if (transactions === null) {
return null;
}
transactions.shift(); // Ignore coinbase
transactions.sort((a: any, b: any) => a.fee - b.fee);
const fees = transactions.map((t: any) => t.fee);
return [
fees[0] ?? 0, // min
fees[Math.max(0, Math.floor(fees.length * 0.1) - 1)] ?? 0, // 10th
fees[Math.max(0, Math.floor(fees.length * 0.25) - 1)] ?? 0, // 25th
fees[Math.max(0, Math.floor(fees.length * 0.5) - 1)] ?? 0, // median
fees[Math.max(0, Math.floor(fees.length * 0.75) - 1)] ?? 0, // 75th
fees[Math.max(0, Math.floor(fees.length * 0.9) - 1)] ?? 0, // 90th
fees[fees.length - 1] ?? 0, // max
];
} catch (e) {
logger.err(`Cannot get block summaries transactions. Reason: ` + (e instanceof Error ? e.message : e));
return null;
}
}
}
export default new BlocksSummariesRepository();

6
backend/src/rpc-api/commands.ts
View File

@ -88,5 +88,7 @@ module.exports = {
verifyTxOutProof: 'verifytxoutproof', // bitcoind v0.11.0+
walletLock: 'walletlock',
walletPassphrase: 'walletpassphrase',
walletPassphraseChange: 'walletpassphrasechange'
}
walletPassphraseChange: 'walletpassphrasechange',
getTxoutSetinfo: 'gettxoutsetinfo',
getIndexInfo: 'getindexinfo',
};

2
docker/README.md
View File

@ -111,6 +111,7 @@ Below we list all settings from `mempool-config.json` and the corresponding over
"ADVANCED_GBT_AUDIT": false,
"ADVANCED_GBT_MEMPOOL": false,
"CPFP_INDEXING": false,
"MAX_BLOCKS_BULK_QUERY": 0,
},
```
@ -141,6 +142,7 @@ Corresponding `docker-compose.yml` overrides:
MEMPOOL_ADVANCED_GBT_AUDIT: ""
MEMPOOL_ADVANCED_GBT_MEMPOOL: ""
MEMPOOL_CPFP_INDEXING: ""
MAX_BLOCKS_BULK_QUERY: ""
...
```

3
docker/backend/mempool-config.json
View File

@ -25,7 +25,8 @@
"AUDIT": __MEMPOOL_AUDIT__,
"ADVANCED_GBT_AUDIT": __MEMPOOL_ADVANCED_GBT_AUDIT__,
"ADVANCED_GBT_MEMPOOL": __MEMPOOL_ADVANCED_GBT_MEMPOOL__,
"CPFP_INDEXING": __MEMPOOL_CPFP_INDEXING__
"CPFP_INDEXING": __MEMPOOL_CPFP_INDEXING__,
"MAX_BLOCKS_BULK_QUERY": __MEMPOOL__MAX_BLOCKS_BULK_QUERY__
},
"CORE_RPC": {
"HOST": "__CORE_RPC_HOST__",

2
docker/backend/start.sh
View File

@ -30,6 +30,7 @@ __MEMPOOL_AUDIT__=${MEMPOOL_AUDIT:=false}
__MEMPOOL_ADVANCED_GBT_AUDIT__=${MEMPOOL_ADVANCED_GBT_AUDIT:=false}
__MEMPOOL_ADVANCED_GBT_MEMPOOL__=${MEMPOOL_ADVANCED_GBT_MEMPOOL:=false}
__MEMPOOL_CPFP_INDEXING__=${MEMPOOL_CPFP_INDEXING:=false}
__MEMPOOL_MAX_BLOCKS_BULK_QUERY__=${MEMPOOL_MAX_BLOCKS_BULK_QUERY:=0}
# CORE_RPC
__CORE_RPC_HOST__=${CORE_RPC_HOST:=127.0.0.1}
@ -142,6 +143,7 @@ sed -i "s!__MEMPOOL_AUDIT__!${__MEMPOOL_AUDIT__}!g" mempool-config.json
sed -i "s!__MEMPOOL_ADVANCED_GBT_MEMPOOL__!${__MEMPOOL_ADVANCED_GBT_MEMPOOL__}!g" mempool-config.json
sed -i "s!__MEMPOOL_ADVANCED_GBT_AUDIT__!${__MEMPOOL_ADVANCED_GBT_AUDIT__}!g" mempool-config.json
sed -i "s!__MEMPOOL_CPFP_INDEXING__!${__MEMPOOL_CPFP_INDEXING__}!g" mempool-config.json
sed -i "s!__MEMPOOL_MAX_BLOCKS_BULK_QUERY__!${__MEMPOOL_MAX_BLOCKS_BULK_QUERY__}!g" mempool-config.json
sed -i "s/__CORE_RPC_HOST__/${__CORE_RPC_HOST__}/g" mempool-config.json
sed -i "s/__CORE_RPC_PORT__/${__CORE_RPC_PORT__}/g" mempool-config.json

1
production/bitcoin.conf
View File

@ -1,6 +1,7 @@
datadir=/bitcoin
server=1
txindex=1
coinstatsindex=1
listen=1
discover=1
par=16