machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into regtest-1

Browse Source
This commit is contained in:
Antoni Spaanderman
2022-02-17 16:05:22 +01:00
committed by GitHub
parent 44173ad5b6 25f4650f5a
commit cfaf8b037a
39 changed files with 1292 additions and 297 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
backend/src/api/bitcoin/bitcoin-api.interface.ts
View File

@@ -4,6 +4,7 @@ export namespace IBitcoinApi {
size: number; // (numeric) Current tx count
bytes: number; // (numeric) Sum of all virtual transaction sizes as defined in BIP 141.
usage: number; // (numeric) Total memory usage for the mempool
total_fee: number; // (numeric) Total fees of transactions in the mempool
maxmempool: number; // (numeric) Maximum memory usage for the mempool
mempoolminfee: number; // (numeric) Minimum fee rate in BTC/kB for tx to be accepted.
minrelaytxfee: number; // (numeric) Current minimum relay fee for transactions

177
backend/src/api/blocks.ts
View File

@@ -10,6 +10,7 @@ import bitcoinClient from './bitcoin/bitcoin-client';
import { IEsploraApi } from './bitcoin/esplora-api.interface';
import poolsRepository from '../repositories/PoolsRepository';
import blocksRepository from '../repositories/BlocksRepository';
import loadingIndicators from './loading-indicators';
class Blocks {
private blocks: BlockExtended[] = [];
@@ -41,7 +42,12 @@ class Blocks {
* @param onlyCoinbase - Set to true if you only need the coinbase transaction
* @returns Promise<TransactionExtended[]>
*/
private async $getTransactionsExtended(blockHash: string, blockHeight: number, onlyCoinbase: boolean): Promise<TransactionExtended[]> {
private async $getTransactionsExtended(
blockHash: string,
blockHeight: number,
onlyCoinbase: boolean,
quiet: boolean = false,
): Promise<TransactionExtended[]> {
const transactions: TransactionExtended[] = [];
const txIds: string[] = await bitcoinApi.$getTxIdsForBlock(blockHash);
@@ -55,9 +61,9 @@ class Blocks {
// optimize here by directly fetching txs in the "outdated" mempool
transactions.push(mempool[txIds[i]]);
transactionsFound++;
} else if (config.MEMPOOL.BACKEND === 'esplora' || memPool.isInSync() || i === 0) {
} else if (config.MEMPOOL.BACKEND === 'esplora' || !memPool.hasPriority() || i === 0) {
// Otherwise we fetch the tx data through backend services (esplora, electrum, core rpc...)
if (i % (Math.round((txIds.length) / 10)) === 0 || i + 1 === txIds.length) { // Avoid log spam
if (!quiet && (i % (Math.round((txIds.length) / 10)) === 0 || i + 1 === txIds.length)) { // Avoid log spam
logger.debug(`Indexing tx ${i + 1} of ${txIds.length} in block #${blockHeight}`);
}
try {
@@ -83,7 +89,9 @@ class Blocks {
}
});
logger.debug(`${transactionsFound} of ${txIds.length} found in mempool. ${transactionsFetched} fetched through backend service.`);
if (!quiet) {
logger.debug(`${transactionsFound} of ${txIds.length} found in mempool. ${transactionsFetched} fetched through backend service.`);
}
return transactions;
}
@@ -94,13 +102,10 @@ class Blocks {
* @param transactions
* @returns BlockExtended
*/
private getBlockExtended(block: IEsploraApi.Block, transactions: TransactionExtended[]): BlockExtended {
const blockExtended: BlockExtended = Object.assign({}, block);
blockExtended.extras = {
reward: transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0),
coinbaseTx: transactionUtils.stripCoinbaseTransaction(transactions[0]),
};
private async $getBlockExtended(block: IEsploraApi.Block, transactions: TransactionExtended[]): Promise<BlockExtended> {
const blockExtended: BlockExtended = Object.assign({extras: {}}, block);
blockExtended.extras.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0);
blockExtended.extras.coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]);
const transactionsTmp = [...transactions];
transactionsTmp.shift();
@@ -111,6 +116,19 @@ class Blocks {
blockExtended.extras.feeRange = transactionsTmp.length > 0 ?
Common.getFeesInRange(transactionsTmp, 8) : [0, 0];
if (Common.indexingEnabled()) {
let pool: PoolTag;
if (blockExtended.extras?.coinbaseTx !== undefined) {
pool = await this.$findBlockMiner(blockExtended.extras?.coinbaseTx);
} else {
pool = await poolsRepository.$getUnknownPool();
}
blockExtended.extras.pool = {
id: pool.id,
name: pool.name
};
}
return blockExtended;
}
@@ -152,20 +170,20 @@ class Blocks {
* Index all blocks metadata for the mining dashboard
*/
public async $generateBlockDatabase() {
if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false || // Bitcoin only
config.MEMPOOL.INDEXING_BLOCKS_AMOUNT === 0 || // Indexing must be enabled
!memPool.isInSync() || // We sync the mempool first
this.blockIndexingStarted === true // Indexing must not already be in progress
if (this.blockIndexingStarted === true ||
!Common.indexingEnabled() ||
memPool.hasPriority()
) {
return;
}
const blockchainInfo = await bitcoinClient.getBlockchainInfo();
if (blockchainInfo.blocks !== blockchainInfo.headers) {
if (blockchainInfo.blocks !== blockchainInfo.headers) { // Wait for node to sync
return;
}
this.blockIndexingStarted = true;
const startedAt = new Date().getTime() / 1000;
try {
let currentBlockHeight = blockchainInfo.blocks;
@@ -180,6 +198,8 @@ class Blocks {
logger.info(`Indexing blocks from #${currentBlockHeight} to #${lastBlockToIndex}`);
const chunkSize = 10000;
let totaIndexed = await blocksRepository.$blockCount(null, null);
let indexedThisRun = 0;
while (currentBlockHeight >= lastBlockToIndex) {
const endBlock = Math.max(0, lastBlockToIndex, currentBlockHeight - chunkSize + 1);
@@ -198,21 +218,19 @@ class Blocks {
break;
}
try {
logger.debug(`Indexing block #${blockHeight}`);
++indexedThisRun;
if (++totaIndexed % 100 === 0 || blockHeight === lastBlockToIndex) {
const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt));
const blockPerSeconds = Math.max(1, Math.round(indexedThisRun / elapsedSeconds));
const progress = Math.round(totaIndexed / indexingBlockAmount * 100);
const timeLeft = Math.round((indexingBlockAmount - totaIndexed) / blockPerSeconds);
logger.debug(`Indexing block #${blockHeight} | ~${blockPerSeconds} blocks/sec | total: ${totaIndexed}/${indexingBlockAmount} (${progress}%) | elapsed: ${elapsedSeconds} seconds | left: ~${timeLeft} seconds`);
}
const blockHash = await bitcoinApi.$getBlockHash(blockHeight);
const block = await bitcoinApi.$getBlock(blockHash);
const transactions = await this.$getTransactionsExtended(blockHash, block.height, true);
const blockExtended = this.getBlockExtended(block, transactions);
let miner: PoolTag;
if (blockExtended?.extras?.coinbaseTx) {
miner = await this.$findBlockMiner(blockExtended.extras.coinbaseTx);
} else {
miner = await poolsRepository.$getUnknownPool();
}
const coinbase: IEsploraApi.Transaction = await bitcoinApi.$getRawTransaction(transactions[0].txid, true);
await blocksRepository.$saveBlockInDatabase(blockExtended, blockHash, coinbase.hex, miner);
const transactions = await this.$getTransactionsExtended(blockHash, block.height, true, true);
const blockExtended = await this.$getBlockExtended(block, transactions);
await blocksRepository.$saveBlockInDatabase(blockExtended);
} catch (e) {
logger.err(`Something went wrong while indexing blocks.` + e);
}
@@ -273,17 +291,10 @@ class Blocks {
const block = await bitcoinApi.$getBlock(blockHash);
const txIds: string[] = await bitcoinApi.$getTxIdsForBlock(blockHash);
const transactions = await this.$getTransactionsExtended(blockHash, block.height, false);
const blockExtended: BlockExtended = this.getBlockExtended(block, transactions);
const coinbase: IEsploraApi.Transaction = await bitcoinApi.$getRawTransaction(transactions[0].txid, true);
const blockExtended: BlockExtended = await this.$getBlockExtended(block, transactions);
if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === true) {
let miner: PoolTag;
if (blockExtended?.extras?.coinbaseTx) {
miner = await this.$findBlockMiner(blockExtended.extras.coinbaseTx);
} else {
miner = await poolsRepository.$getUnknownPool();
}
await blocksRepository.$saveBlockInDatabase(blockExtended, blockHash, coinbase.hex, miner);
if (Common.indexingEnabled()) {
await blocksRepository.$saveBlockInDatabase(blockExtended);
}
if (block.height % 2016 === 0) {
@@ -300,12 +311,98 @@ class Blocks {
if (this.newBlockCallbacks.length) {
this.newBlockCallbacks.forEach((cb) => cb(blockExtended, txIds, transactions));
}
if (memPool.isInSync()) {
if (!memPool.hasPriority()) {
diskCache.$saveCacheToDisk();
}
}
}
/**
* Index a block if it's missing from the database. Returns the block after indexing
*/
public async $indexBlock(height: number): Promise<BlockExtended> {
const dbBlock = await blocksRepository.$getBlockByHeight(height);
if (dbBlock != null) {
return this.prepareBlock(dbBlock);
}
const blockHash = await bitcoinApi.$getBlockHash(height);
const block = await bitcoinApi.$getBlock(blockHash);
const transactions = await this.$getTransactionsExtended(blockHash, block.height, true);
const blockExtended = await this.$getBlockExtended(block, transactions);
await blocksRepository.$saveBlockInDatabase(blockExtended);
return blockExtended;
}
public async $getBlocksExtras(fromHeight: number): Promise<BlockExtended[]> {
try {
loadingIndicators.setProgress('blocks', 0);
let currentHeight = fromHeight ? fromHeight : this.getCurrentBlockHeight();
const returnBlocks: BlockExtended[] = [];
if (currentHeight < 0) {
return returnBlocks;
}
// Check if block height exist in local cache to skip the hash lookup
const blockByHeight = this.getBlocks().find((b) => b.height === currentHeight);
let startFromHash: string | null = null;
if (blockByHeight) {
startFromHash = blockByHeight.id;
} else {
startFromHash = await bitcoinApi.$getBlockHash(currentHeight);
}
let nextHash = startFromHash;
for (let i = 0; i < 10 && currentHeight >= 0; i++) {
let block = this.getBlocks().find((b) => b.height === currentHeight);
if (!block && Common.indexingEnabled()) {
block = this.prepareBlock(await this.$indexBlock(currentHeight));
} else if (!block) {
block = this.prepareBlock(await bitcoinApi.$getBlock(nextHash));
}
returnBlocks.push(block);
nextHash = block.previousblockhash;
loadingIndicators.setProgress('blocks', i / 10 * 100);
currentHeight--;
}
return returnBlocks;
} catch (e) {
loadingIndicators.setProgress('blocks', 100);
throw e;
}
}
private prepareBlock(block: any): BlockExtended {
return <BlockExtended>{
id: block.id ?? block.hash, // hash for indexed block
timestamp: block?.timestamp ?? block?.blockTimestamp, // blockTimestamp for indexed block
height: block?.height,
version: block?.version,
bits: block?.bits,
nonce: block?.nonce,
difficulty: block?.difficulty,
merkle_root: block?.merkle_root,
tx_count: block?.tx_count,
size: block?.size,
weight: block?.weight,
previousblockhash: block?.previousblockhash,
extras: {
medianFee: block?.medianFee,
feeRange: block?.feeRange ?? [], // TODO
reward: block?.reward,
pool: block?.extras?.pool ?? (block?.pool_id ? {
id: block?.pool_id,
name: block?.pool_name,
} : undefined),
}
};
}
public getLastDifficultyAdjustmentTime(): number {
return this.lastDifficultyAdjustmentTime;
}

23
backend/src/api/common.ts
View File

@@ -154,4 +154,27 @@ export class Common {
});
return parents;
}
static getSqlInterval(interval: string | null): string | null {
switch (interval) {
case '24h': return '1 DAY';
case '3d': return '3 DAY';
case '1w': return '1 WEEK';
case '1m': return '1 MONTH';
case '3m': return '3 MONTH';
case '6m': return '6 MONTH';
case '1y': return '1 YEAR';
case '2y': return '2 YEAR';
case '3y': return '3 YEAR';
default: return null;
}
}
static indexingEnabled(): boolean {
return (
['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) &&
config.DATABASE.ENABLED === true &&
config.MEMPOOL.INDEXING_BLOCKS_AMOUNT != 0
);
}
}

28
backend/src/api/database-migration.ts
View File

@@ -6,7 +6,7 @@ import logger from '../logger';
const sleep = (ms: number) => new Promise(res => setTimeout(res, ms));
class DatabaseMigration {
private static currentVersion = 4;
private static currentVersion = 6;
private queryTimeout = 120000;
private statisticsAddedIndexed = false;
@@ -76,6 +76,7 @@ class DatabaseMigration {
private async $createMissingTablesAndIndexes(databaseSchemaVersion: number) {
await this.$setStatisticsAddedIndexedFlag(databaseSchemaVersion);
const isBitcoin = ['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK);
const connection = await DB.pool.getConnection();
try {
await this.$executeQuery(connection, this.getCreateElementsTableQuery(), await this.$checkIfTableExists('elements_pegs'));
@@ -90,6 +91,31 @@ class DatabaseMigration {
await this.$executeQuery(connection, 'DROP table IF EXISTS blocks;');
await this.$executeQuery(connection, this.getCreateBlocksTableQuery(), await this.$checkIfTableExists('blocks'));
}
if (databaseSchemaVersion < 5 && isBitcoin === true) {
await this.$executeQuery(connection, 'TRUNCATE blocks;'); // Need to re-index
await this.$executeQuery(connection, 'ALTER TABLE blocks ADD `reward` double unsigned NOT NULL DEFAULT "0"');
}
if (databaseSchemaVersion < 6 && isBitcoin === true) {
await this.$executeQuery(connection, 'TRUNCATE blocks;'); // Need to re-index
// Cleanup original blocks fields type
await this.$executeQuery(connection, 'ALTER TABLE blocks MODIFY `height` integer unsigned NOT NULL DEFAULT "0"');
await this.$executeQuery(connection, 'ALTER TABLE blocks MODIFY `tx_count` smallint unsigned NOT NULL DEFAULT "0"');
await this.$executeQuery(connection, 'ALTER TABLE blocks MODIFY `size` integer unsigned NOT NULL DEFAULT "0"');
await this.$executeQuery(connection, 'ALTER TABLE blocks MODIFY `weight` integer unsigned NOT NULL DEFAULT "0"');
await this.$executeQuery(connection, 'ALTER TABLE blocks MODIFY `difficulty` double NOT NULL DEFAULT "0"');
// We also fix the pools.id type so we need to drop/re-create the foreign key
await this.$executeQuery(connection, 'ALTER TABLE blocks DROP FOREIGN KEY IF EXISTS `blocks_ibfk_1`');
await this.$executeQuery(connection, 'ALTER TABLE pools MODIFY `id` smallint unsigned AUTO_INCREMENT');
await this.$executeQuery(connection, 'ALTER TABLE blocks MODIFY `pool_id` smallint unsigned NULL');
await this.$executeQuery(connection, 'ALTER TABLE blocks ADD FOREIGN KEY (`pool_id`) REFERENCES `pools` (`id`)');
// Add new block indexing fields
await this.$executeQuery(connection, 'ALTER TABLE blocks ADD `version` integer unsigned NOT NULL DEFAULT "0"');
await this.$executeQuery(connection, 'ALTER TABLE blocks ADD `bits` integer unsigned NOT NULL DEFAULT "0"');
await this.$executeQuery(connection, 'ALTER TABLE blocks ADD `nonce` bigint unsigned NOT NULL DEFAULT "0"');
await this.$executeQuery(connection, 'ALTER TABLE blocks ADD `merkle_root` varchar(65) NOT NULL DEFAULT ""');
await this.$executeQuery(connection, 'ALTER TABLE blocks ADD `previous_block_hash` varchar(65) NULL');
}
connection.release();
} catch (e) {
connection.release();

21
backend/src/api/mempool.ts
View File

@@ -13,8 +13,9 @@ class Mempool {
private static WEBSOCKET_REFRESH_RATE_MS = 10000;
private static LAZY_DELETE_AFTER_SECONDS = 30;
private inSync: boolean = false;
private mempoolCacheDelta: number = -1;
private mempoolCache: { [txId: string]: TransactionExtended } = {};
private mempoolInfo: IBitcoinApi.MempoolInfo = { loaded: false, size: 0, bytes: 0, usage: 0,
private mempoolInfo: IBitcoinApi.MempoolInfo = { loaded: false, size: 0, bytes: 0, usage: 0, total_fee: 0,
maxmempool: 300000000, mempoolminfee: 0.00001000, minrelaytxfee: 0.00001000 };
private mempoolChangedCallback: ((newMempool: {[txId: string]: TransactionExtended; }, newTransactions: TransactionExtended[],
deletedTransactions: TransactionExtended[]) => void) | undefined;
@@ -32,6 +33,17 @@ class Mempool {
setInterval(this.deleteExpiredTransactions.bind(this), 20000);
}
/**
* Return true if we should leave resources available for mempool tx caching
*/
public hasPriority(): boolean {
if (this.inSync) {
return false;
} else {
return this.mempoolCacheDelta == -1 || this.mempoolCacheDelta > 25;
}
}
public isInSync(): boolean {
return this.inSync;
}
@@ -100,6 +112,8 @@ class Mempool {
const diff = transactions.length - currentMempoolSize;
const newTransactions: TransactionExtended[] = [];
this.mempoolCacheDelta = Math.abs(diff);
if (!this.inSync) {
loadingIndicators.setProgress('mempool', Object.keys(this.mempoolCache).length / transactions.length * 100);
}
@@ -168,13 +182,14 @@ class Mempool {
const newTransactionsStripped = newTransactions.map((tx) => Common.stripTransaction(tx));
this.latestTransactions = newTransactionsStripped.concat(this.latestTransactions).slice(0, 6);
const syncedThreshold = 0.99; // If we synced 99% of the mempool tx count, consider we're synced
if (!this.inSync && Object.keys(this.mempoolCache).length >= transactions.length * syncedThreshold) {
if (!this.inSync && transactions.length === Object.keys(this.mempoolCache).length) {
this.inSync = true;
logger.notice('The mempool is now in sync!');
loadingIndicators.setProgress('mempool', 100);
}
this.mempoolCacheDelta = Math.abs(transactions.length - Object.keys(this.mempoolCache).length);
if (this.mempoolChangedCallback && (hasChange || deletedTransactions.length)) {
this.mempoolChangedCallback(this.mempoolCache, newTransactions, deletedTransactions);
}

52
backend/src/api/mining.ts
View File

@@ -11,24 +11,10 @@ class Mining {
* Generate high level overview of the pool ranks and general stats
*/
public async $getPoolsStats(interval: string | null) : Promise<object> {
let sqlInterval: string | null = null;
switch (interval) {
case '24h': sqlInterval = '1 DAY'; break;
case '3d': sqlInterval = '3 DAY'; break;
case '1w': sqlInterval = '1 WEEK'; break;
case '1m': sqlInterval = '1 MONTH'; break;
case '3m': sqlInterval = '3 MONTH'; break;
case '6m': sqlInterval = '6 MONTH'; break;
case '1y': sqlInterval = '1 YEAR'; break;
case '2y': sqlInterval = '2 YEAR'; break;
case '3y': sqlInterval = '3 YEAR'; break;
default: sqlInterval = null; break;
}
const poolsStatistics = {};
const poolsInfo: PoolInfo[] = await PoolsRepository.$getPoolsInfo(sqlInterval);
const emptyBlocks: EmptyBlocks[] = await BlocksRepository.$countEmptyBlocks(sqlInterval);
const poolsInfo: PoolInfo[] = await PoolsRepository.$getPoolsInfo(interval);
const emptyBlocks: EmptyBlocks[] = await BlocksRepository.$getEmptyBlocks(null, interval);
const poolsStats: PoolStats[] = [];
let rank = 1;
@@ -55,7 +41,7 @@ class Mining {
const oldestBlock = new Date(await BlocksRepository.$oldestBlockTimestamp());
poolsStatistics['oldestIndexedBlockTimestamp'] = oldestBlock.getTime();
const blockCount: number = await BlocksRepository.$blockCount(sqlInterval);
const blockCount: number = await BlocksRepository.$blockCount(null, interval);
poolsStatistics['blockCount'] = blockCount;
const blockHeightTip = await bitcoinClient.getBlockCount();
@@ -64,6 +50,38 @@ class Mining {
return poolsStatistics;
}
/**
* Get all mining pool stats for a pool
*/
public async $getPoolStat(interval: string | null, poolId: number): Promise<object> {
const pool = await PoolsRepository.$getPool(poolId);
if (!pool) {
throw new Error(`This mining pool does not exist`);
}
const blockCount: number = await BlocksRepository.$blockCount(poolId, interval);
const emptyBlocks: EmptyBlocks[] = await BlocksRepository.$getEmptyBlocks(poolId, interval);
return {
pool: pool,
blockCount: blockCount,
emptyBlocks: emptyBlocks,
};
}
/**
* Return the historical difficulty adjustments and oldest indexed block timestamp
*/
public async $getHistoricalDifficulty(interval: string | null): Promise<object> {
const difficultyAdjustments = await BlocksRepository.$getBlocksDifficulty(interval);
const oldestBlock = new Date(await BlocksRepository.$oldestBlockTimestamp());
return {
adjustments: difficultyAdjustments,
oldestIndexedBlockTimestamp: oldestBlock.getTime(),
}
}
}
export default new Mining();