machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into regtest-1

Browse Source 
for cla bot
This commit is contained in:
Antoni Spaanderman
2022-02-04 12:34:42 +01:00
parent f68ac944ed 5c1fc7344d
commit d08d2a16d3
66 changed files with 3112 additions and 1265 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
backend/README.md Normal file
View File

@@ -0,0 +1,22 @@
# Setup backend watchers
The backend is static. Typescript scripts are compiled into the `dist` folder and served through a node web server.
You can avoid the manual shutdown/recompile/restart command line cycle by using a watcher.
Make sure you are in the `backend` directory `cd backend`.
1. Install nodemon and ts-node
```
sudo npm install -g ts-node nodemon
```
2. Run the watcher
> Note: You can find your npm global binary folder using `npm -g bin`, where nodemon will be installed.
```
nodemon src/index.ts --ignore cache/ --ignore pools.json
```

5
backend/mempool-config.sample.json
View File

@@ -12,9 +12,12 @@
"BLOCK_WEIGHT_UNITS": 4000000,
"INITIAL_BLOCKS_AMOUNT": 8,
"MEMPOOL_BLOCKS_AMOUNT": 8,
"INDEXING_BLOCKS_AMOUNT": 1100,
"PRICE_FEED_UPDATE_INTERVAL": 3600,
"USE_SECOND_NODE_FOR_MINFEE": false,
"EXTERNAL_ASSETS": []
"EXTERNAL_ASSETS": [
"https://mempool.space/resources/pools.json"
]
},
"CORE_RPC": {
"HOST": "127.0.0.1",

19
backend/src/api/bitcoin/bitcoin-api.ts
View File

@@ -112,14 +112,25 @@ class BitcoinApi implements AbstractBitcoinApi {
const outSpends: IEsploraApi.Outspend[] = [];
const tx = await this.$getRawTransaction(txId, true, false);
for (let i = 0; i < tx.vout.length; i++) {
const txOut = await this.bitcoindClient.getTxOut(txId, i);
outSpends.push({
spent: txOut === null,
});
if (tx.status && tx.status.block_height == 0) {
outSpends.push({
spent: false
});
} else {
const txOut = await this.bitcoindClient.getTxOut(txId, i);
outSpends.push({
spent: txOut === null,
});
}
}
return outSpends;
}
$getEstimatedHashrate(blockHeight: number): Promise<number> {
// 120 is the default block span in Core
return this.bitcoindClient.getNetworkHashPs(120, blockHeight);
}
protected async $convertTransaction(transaction: IBitcoinApi.Transaction, addPrevout: boolean): Promise<IEsploraApi.Transaction> {
let esploraTransaction: IEsploraApi.Transaction = {
txid: transaction.txid,

231
backend/src/api/blocks.ts
View File

@@ -2,11 +2,14 @@ import config from '../config';
import bitcoinApi from './bitcoin/bitcoin-api-factory';
import logger from '../logger';
import memPool from './mempool';
import { BlockExtended, TransactionExtended } from '../mempool.interfaces';
import { BlockExtended, PoolTag, TransactionExtended, TransactionMinerInfo } from '../mempool.interfaces';
import { Common } from './common';
import diskCache from './disk-cache';
import transactionUtils from './transaction-utils';
import bitcoinClient from './bitcoin/bitcoin-client';
import { IEsploraApi } from './bitcoin/esplora-api.interface';
import poolsRepository from '../repositories/PoolsRepository';
import blocksRepository from '../repositories/BlocksRepository';
class Blocks {
private blocks: BlockExtended[] = [];
@@ -15,6 +18,7 @@ class Blocks {
private lastDifficultyAdjustmentTime = 0;
private previousDifficultyRetarget = 0;
private newBlockCallbacks: ((block: BlockExtended, txIds: string[], transactions: TransactionExtended[]) => void)[] = [];
private blockIndexingStarted = false;
constructor() { }
@@ -30,6 +34,186 @@ class Blocks {
this.newBlockCallbacks.push(fn);
}
/**
* Return the list of transaction for a block
* @param blockHash
* @param blockHeight
* @param onlyCoinbase - Set to true if you only need the coinbase transaction
* @returns Promise<TransactionExtended[]>
*/
private async $getTransactionsExtended(blockHash: string, blockHeight: number, onlyCoinbase: boolean): Promise<TransactionExtended[]> {
const transactions: TransactionExtended[] = [];
const txIds: string[] = await bitcoinApi.$getTxIdsForBlock(blockHash);
const mempool = memPool.getMempool();
let transactionsFound = 0;
let transactionsFetched = 0;
for (let i = 0; i < txIds.length; i++) {
if (mempool[txIds[i]]) {
// We update blocks before the mempool (index.ts), therefore we can
// optimize here by directly fetching txs in the "outdated" mempool
transactions.push(mempool[txIds[i]]);
transactionsFound++;
} else if (config.MEMPOOL.BACKEND === 'esplora' || memPool.isInSync() || i === 0) {
// Otherwise we fetch the tx data through backend services (esplora, electrum, core rpc...)
if (i % (Math.round((txIds.length) / 10)) === 0 || i + 1 === txIds.length) { // Avoid log spam
logger.debug(`Indexing tx ${i + 1} of ${txIds.length} in block #${blockHeight}`);
}
try {
const tx = await transactionUtils.$getTransactionExtended(txIds[i]);
transactions.push(tx);
transactionsFetched++;
} catch (e) {
logger.debug('Error fetching block tx: ' + (e instanceof Error ? e.message : e));
if (i === 0) {
throw new Error('Failed to fetch Coinbase transaction: ' + txIds[i]);
}
}
}
if (onlyCoinbase === true) {
break; // Fetch the first transaction and exit
}
}
transactions.forEach((tx) => {
if (!tx.cpfpChecked) {
Common.setRelativesAndGetCpfpInfo(tx, mempool); // Child Pay For Parent
}
});
logger.debug(`${transactionsFound} of ${txIds.length} found in mempool. ${transactionsFetched} fetched through backend service.`);
return transactions;
}
/**
* Return a block with additional data (reward, coinbase, fees...)
* @param block
* @param transactions
* @returns BlockExtended
*/
private getBlockExtended(block: IEsploraApi.Block, transactions: TransactionExtended[]): BlockExtended {
const blockExtended: BlockExtended = Object.assign({}, block);
blockExtended.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0);
blockExtended.coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]);
const transactionsTmp = [...transactions];
transactionsTmp.shift();
transactionsTmp.sort((a, b) => b.effectiveFeePerVsize - a.effectiveFeePerVsize);
blockExtended.medianFee = transactionsTmp.length > 0 ? Common.median(transactionsTmp.map((tx) => tx.effectiveFeePerVsize)) : 0;
blockExtended.feeRange = transactionsTmp.length > 0 ? Common.getFeesInRange(transactionsTmp, 8) : [0, 0];
return blockExtended;
}
/**
* Try to find which miner found the block
* @param txMinerInfo
* @returns
*/
private async $findBlockMiner(txMinerInfo: TransactionMinerInfo | undefined): Promise<PoolTag> {
if (txMinerInfo === undefined || txMinerInfo.vout.length < 1) {
return await poolsRepository.$getUnknownPool();
}
const asciiScriptSig = transactionUtils.hex2ascii(txMinerInfo.vin[0].scriptsig);
const address = txMinerInfo.vout[0].scriptpubkey_address;
const pools: PoolTag[] = await poolsRepository.$getPools();
for (let i = 0; i < pools.length; ++i) {
if (address !== undefined) {
const addresses: string[] = JSON.parse(pools[i].addresses);
if (addresses.indexOf(address) !== -1) {
return pools[i];
}
}
const regexes: string[] = JSON.parse(pools[i].regexes);
for (let y = 0; y < regexes.length; ++y) {
const match = asciiScriptSig.match(regexes[y]);
if (match !== null) {
return pools[i];
}
}
}
return await poolsRepository.$getUnknownPool();
}
/**
* Index all blocks metadata for the mining dashboard
*/
public async $generateBlockDatabase() {
if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false || // Bitcoin only
config.MEMPOOL.INDEXING_BLOCKS_AMOUNT === 0 || // Indexing must be enabled
!memPool.isInSync() || // We sync the mempool first
this.blockIndexingStarted === true // Indexing must not already be in progress
) {
return;
}
const blockchainInfo = await bitcoinClient.getBlockchainInfo();
if (blockchainInfo.blocks !== blockchainInfo.headers) {
return;
}
this.blockIndexingStarted = true;
try {
let currentBlockHeight = blockchainInfo.blocks;
let indexingBlockAmount = config.MEMPOOL.INDEXING_BLOCKS_AMOUNT;
if (indexingBlockAmount <= -1) {
indexingBlockAmount = currentBlockHeight + 1;
}
const lastBlockToIndex = Math.max(0, currentBlockHeight - indexingBlockAmount + 1);
logger.info(`Indexing blocks from #${currentBlockHeight} to #${lastBlockToIndex}`);
const chunkSize = 10000;
while (currentBlockHeight >= lastBlockToIndex) {
const endBlock = Math.max(0, lastBlockToIndex, currentBlockHeight - chunkSize + 1);
const missingBlockHeights: number[] = await blocksRepository.$getMissingBlocksBetweenHeights(
currentBlockHeight, endBlock);
if (missingBlockHeights.length <= 0) {
logger.debug(`No missing blocks between #${currentBlockHeight} to #${endBlock}`);
currentBlockHeight -= chunkSize;
continue;
}
logger.debug(`Indexing ${missingBlockHeights.length} blocks from #${currentBlockHeight} to #${endBlock}`);
for (const blockHeight of missingBlockHeights) {
if (blockHeight < lastBlockToIndex) {
break;
}
try {
logger.debug(`Indexing block #${blockHeight}`);
const blockHash = await bitcoinApi.$getBlockHash(blockHeight);
const block = await bitcoinApi.$getBlock(blockHash);
const transactions = await this.$getTransactionsExtended(blockHash, block.height, true);
const blockExtended = this.getBlockExtended(block, transactions);
const miner = await this.$findBlockMiner(blockExtended.coinbaseTx);
const coinbase: IEsploraApi.Transaction = await bitcoinApi.$getRawTransaction(transactions[0].txid, true);
await blocksRepository.$saveBlockInDatabase(blockExtended, blockHash, coinbase.hex, miner);
} catch (e) {
logger.err(`Something went wrong while indexing blocks.` + e);
}
}
currentBlockHeight -= chunkSize;
}
logger.info('Block indexing completed');
} catch (e) {
logger.err('An error occured in $generateBlockDatabase(). Skipping block indexing. ' + e);
console.log(e);
}
}
public async $updateBlocks() {
const blockHeightTip = await bitcoinApi.$getBlockHeightTip();
@@ -72,49 +256,18 @@ class Blocks {
logger.debug(`New block found (#${this.currentBlockHeight})!`);
}
const transactions: TransactionExtended[] = [];
const blockHash = await bitcoinApi.$getBlockHash(this.currentBlockHeight);
const block = await bitcoinApi.$getBlock(blockHash);
const txIds: string[] = await bitcoinApi.$getTxIdsForBlock(blockHash);
const transactions = await this.$getTransactionsExtended(blockHash, block.height, false);
const blockExtended: BlockExtended = this.getBlockExtended(block, transactions);
const coinbase: IEsploraApi.Transaction = await bitcoinApi.$getRawTransaction(transactions[0].txid, true);
const mempool = memPool.getMempool();
let transactionsFound = 0;
for (let i = 0; i < txIds.length; i++) {
if (mempool[txIds[i]]) {
transactions.push(mempool[txIds[i]]);
transactionsFound++;
} else if (config.MEMPOOL.BACKEND === 'esplora' || memPool.isInSync() || i === 0) {
logger.debug(`Fetching block tx ${i} of ${txIds.length}`);
try {
const tx = await transactionUtils.$getTransactionExtended(txIds[i]);
transactions.push(tx);
} catch (e) {
logger.debug('Error fetching block tx: ' + (e instanceof Error ? e.message : e));
if (i === 0) {
throw new Error('Failed to fetch Coinbase transaction: ' + txIds[i]);
}
}
}
if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === true) {
const miner = await this.$findBlockMiner(blockExtended.coinbaseTx);
await blocksRepository.$saveBlockInDatabase(blockExtended, blockHash, coinbase.hex, miner);
}
transactions.forEach((tx) => {
if (!tx.cpfpChecked) {
Common.setRelativesAndGetCpfpInfo(tx, mempool);
}
});
logger.debug(`${transactionsFound} of ${txIds.length} found in mempool. ${txIds.length - transactionsFound} not found.`);
const blockExtended: BlockExtended = Object.assign({}, block);
blockExtended.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0);
blockExtended.coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]);
transactions.shift();
transactions.sort((a, b) => b.effectiveFeePerVsize - a.effectiveFeePerVsize);
blockExtended.medianFee = transactions.length > 0 ? Common.median(transactions.map((tx) => tx.effectiveFeePerVsize)) : 0;
blockExtended.feeRange = transactions.length > 0 ? Common.getFeesInRange(transactions, 8) : [0, 0];
if (block.height % 2016 === 0) {
this.previousDifficultyRetarget = (block.difficulty - this.currentDifficulty) / this.currentDifficulty * 100;
this.lastDifficultyAdjustmentTime = block.timestamp;
@@ -132,6 +285,8 @@ class Blocks {
if (memPool.isInSync()) {
diskCache.$saveCacheToDisk();
}
return;
}
}

43
backend/src/api/database-migration.ts
View File

@@ -3,10 +3,10 @@ import config from '../config';
import { DB } from '../database';
import logger from '../logger';
const sleep = (ms: number) => new Promise( res => setTimeout(res, ms));
const sleep = (ms: number) => new Promise(res => setTimeout(res, ms));
class DatabaseMigration {
private static currentVersion = 2;
private static currentVersion = 4;
private queryTimeout = 120000;
private statisticsAddedIndexed = false;
@@ -83,6 +83,13 @@ class DatabaseMigration {
if (databaseSchemaVersion < 2 && this.statisticsAddedIndexed === false) {
await this.$executeQuery(connection, `CREATE INDEX added ON statistics (added);`);
}
if (databaseSchemaVersion < 3) {
await this.$executeQuery(connection, this.getCreatePoolsTableQuery(), await this.$checkIfTableExists('pools'));
}
if (databaseSchemaVersion < 4) {
await this.$executeQuery(connection, 'DROP table IF EXISTS blocks;');
await this.$executeQuery(connection, this.getCreateBlocksTableQuery(), await this.$checkIfTableExists('blocks'));
}
connection.release();
} catch (e) {
connection.release();
@@ -197,7 +204,6 @@ class DatabaseMigration {
const connection = await DB.pool.getConnection();
try {
await this.$executeQuery(connection, 'START TRANSACTION;');
await this.$executeQuery(connection, 'SET autocommit = 0;');
for (const query of transactionQueries) {
await this.$executeQuery(connection, query);
}
@@ -335,6 +341,37 @@ class DatabaseMigration {
final_tx int(11) NOT NULL
) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
}
private getCreatePoolsTableQuery(): string {
return `CREATE TABLE IF NOT EXISTS pools (
id int(11) NOT NULL AUTO_INCREMENT,
name varchar(50) NOT NULL,
link varchar(255) NOT NULL,
addresses text NOT NULL,
regexes text NOT NULL,
PRIMARY KEY (id)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;`;
}
private getCreateBlocksTableQuery(): string {
return `CREATE TABLE IF NOT EXISTS blocks (
height int(11) unsigned NOT NULL,
hash varchar(65) NOT NULL,
blockTimestamp timestamp NOT NULL,
size int(11) unsigned NOT NULL,
weight int(11) unsigned NOT NULL,
tx_count int(11) unsigned NOT NULL,
coinbase_raw text,
difficulty bigint(20) unsigned NOT NULL,
pool_id int(11) DEFAULT -1,
fees double unsigned NOT NULL,
fee_span json NOT NULL,
median_fee double unsigned NOT NULL,
PRIMARY KEY (height),
INDEX (pool_id),
FOREIGN KEY (pool_id) REFERENCES pools (id)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
}
}
export default new DatabaseMigration();

69
backend/src/api/mining.ts Normal file
View File

@@ -0,0 +1,69 @@
import { PoolInfo, PoolStats } from '../mempool.interfaces';
import BlocksRepository, { EmptyBlocks } from '../repositories/BlocksRepository';
import PoolsRepository from '../repositories/PoolsRepository';
import bitcoinClient from './bitcoin/bitcoin-client';
class Mining {
constructor() {
}
/**
* Generate high level overview of the pool ranks and general stats
*/
public async $getPoolsStats(interval: string | null) : Promise<object> {
let sqlInterval: string | null = null;
switch (interval) {
case '24h': sqlInterval = '1 DAY'; break;
case '3d': sqlInterval = '3 DAY'; break;
case '1w': sqlInterval = '1 WEEK'; break;
case '1m': sqlInterval = '1 MONTH'; break;
case '3m': sqlInterval = '3 MONTH'; break;
case '6m': sqlInterval = '6 MONTH'; break;
case '1y': sqlInterval = '1 YEAR'; break;
case '2y': sqlInterval = '2 YEAR'; break;
case '3y': sqlInterval = '3 YEAR'; break;
default: sqlInterval = null; break;
}
const poolsStatistics = {};
const poolsInfo: PoolInfo[] = await PoolsRepository.$getPoolsInfo(sqlInterval);
const emptyBlocks: EmptyBlocks[] = await BlocksRepository.$countEmptyBlocks(sqlInterval);
const poolsStats: PoolStats[] = [];
let rank = 1;
poolsInfo.forEach((poolInfo: PoolInfo) => {
const poolStat: PoolStats = {
poolId: poolInfo.poolId, // mysql row id
name: poolInfo.name,
link: poolInfo.link,
blockCount: poolInfo.blockCount,
rank: rank++,
emptyBlocks: 0,
}
for (let i = 0; i < emptyBlocks.length; ++i) {
if (emptyBlocks[i].poolId === poolInfo.poolId) {
poolStat.emptyBlocks++;
}
}
poolsStats.push(poolStat);
});
poolsStatistics['pools'] = poolsStats;
const oldestBlock = new Date(await BlocksRepository.$oldestBlockTimestamp());
poolsStatistics['oldestIndexedBlockTimestamp'] = oldestBlock.getTime();
const blockCount: number = await BlocksRepository.$blockCount(sqlInterval);
poolsStatistics['blockCount'] = blockCount;
const blockHeightTip = await bitcoinClient.getBlockCount();
const lastBlockHashrate = await bitcoinClient.getNetworkHashPs(120, blockHeightTip);
poolsStatistics['lastEstimatedHashrate'] = lastBlockHashrate;
return poolsStatistics;
}
}
export default new Mining();

173
backend/src/api/pools-parser.ts Normal file
View File

@@ -0,0 +1,173 @@
import { readFileSync } from 'fs';
import { DB } from '../database';
import logger from '../logger';
import config from '../config';
interface Pool {
name: string;
link: string;
regexes: string[];
addresses: string[];
}
class PoolsParser {
/**
* Parse the pools.json file, consolidate the data and dump it into the database
*/
public async migratePoolsJson() {
if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) {
return;
}
logger.debug('Importing pools.json to the database, open ./pools.json');
let poolsJson: object = {};
try {
const fileContent: string = readFileSync('./pools.json', 'utf8');
poolsJson = JSON.parse(fileContent);
} catch (e) {
logger.err('Unable to open ./pools.json, does the file exist?');
await this.insertUnknownPool();
return;
}
// First we save every entries without paying attention to pool duplication
const poolsDuplicated: Pool[] = [];
logger.debug('Parse coinbase_tags');
const coinbaseTags = Object.entries(poolsJson['coinbase_tags']);
for (let i = 0; i < coinbaseTags.length; ++i) {
poolsDuplicated.push({
'name': (<Pool>coinbaseTags[i][1]).name,
'link': (<Pool>coinbaseTags[i][1]).link,
'regexes': [coinbaseTags[i][0]],
'addresses': [],
});
}
logger.debug('Parse payout_addresses');
const addressesTags = Object.entries(poolsJson['payout_addresses']);
for (let i = 0; i < addressesTags.length; ++i) {
poolsDuplicated.push({
'name': (<Pool>addressesTags[i][1]).name,
'link': (<Pool>addressesTags[i][1]).link,
'regexes': [],
'addresses': [addressesTags[i][0]],
});
}
// Then, we find unique mining pool names
logger.debug('Identify unique mining pools');
const poolNames: string[] = [];
for (let i = 0; i < poolsDuplicated.length; ++i) {
if (poolNames.indexOf(poolsDuplicated[i].name) === -1) {
poolNames.push(poolsDuplicated[i].name);
}
}
logger.debug(`Found ${poolNames.length} unique mining pools`);
// Get existing pools from the db
const connection = await DB.pool.getConnection();
let existingPools;
try {
[existingPools] = await connection.query<any>({ sql: 'SELECT * FROM pools;', timeout: 120000 });
} catch (e) {
logger.err('Unable to get existing pools from the database, skipping pools.json import');
connection.release();
return;
}
// Finally, we generate the final consolidated pools data
const finalPoolDataAdd: Pool[] = [];
const finalPoolDataUpdate: Pool[] = [];
for (let i = 0; i < poolNames.length; ++i) {
let allAddresses: string[] = [];
let allRegexes: string[] = [];
const match = poolsDuplicated.filter((pool: Pool) => pool.name === poolNames[i]);
for (let y = 0; y < match.length; ++y) {
allAddresses = allAddresses.concat(match[y].addresses);
allRegexes = allRegexes.concat(match[y].regexes);
}
const finalPoolName = poolNames[i].replace(`'`, `''`); // To support single quote in names when doing db queries
if (existingPools.find((pool) => pool.name === poolNames[i]) !== undefined) {
logger.debug(`Update '${finalPoolName}' mining pool`);
finalPoolDataUpdate.push({
'name': finalPoolName,
'link': match[0].link,
'regexes': allRegexes,
'addresses': allAddresses,
});
} else {
logger.debug(`Add '${finalPoolName}' mining pool`);
finalPoolDataAdd.push({
'name': finalPoolName,
'link': match[0].link,
'regexes': allRegexes,
'addresses': allAddresses,
});
}
}
logger.debug(`Update pools table now`);
// Add new mining pools into the database
let queryAdd: string = 'INSERT INTO pools(name, link, regexes, addresses) VALUES ';
for (let i = 0; i < finalPoolDataAdd.length; ++i) {
queryAdd += `('${finalPoolDataAdd[i].name}', '${finalPoolDataAdd[i].link}',
'${JSON.stringify(finalPoolDataAdd[i].regexes)}', '${JSON.stringify(finalPoolDataAdd[i].addresses)}'),`;
}
queryAdd = queryAdd.slice(0, -1) + ';';
// Add new mining pools into the database
const updateQueries: string[] = [];
for (let i = 0; i < finalPoolDataUpdate.length; ++i) {
updateQueries.push(`
UPDATE pools
SET name='${finalPoolDataUpdate[i].name}', link='${finalPoolDataUpdate[i].link}',
regexes='${JSON.stringify(finalPoolDataUpdate[i].regexes)}', addresses='${JSON.stringify(finalPoolDataUpdate[i].addresses)}'
WHERE name='${finalPoolDataUpdate[i].name}'
;`);
}
try {
if (finalPoolDataAdd.length > 0) {
await connection.query<any>({ sql: queryAdd, timeout: 120000 });
}
for (const query of updateQueries) {
await connection.query<any>({ sql: query, timeout: 120000 });
}
await this.insertUnknownPool();
connection.release();
logger.info('Mining pools.json import completed');
} catch (e) {
connection.release();
logger.err(`Unable to import pools in the database!`);
throw e;
}
}
/**
* Manually add the 'unknown pool'
*/
private async insertUnknownPool() {
const connection = await DB.pool.getConnection();
try {
const [rows]: any[] = await connection.query({ sql: 'SELECT name from pools where name="Unknown"', timeout: 120000 });
if (rows.length === 0) {
logger.debug('Manually inserting "Unknown" mining pool into the databse');
await connection.query({
sql: `INSERT INTO pools(name, link, regexes, addresses)
VALUES("Unknown", "https://learnmeabitcoin.com/technical/coinbase-transaction", "[]", "[]");
`});
}
} catch (e) {
logger.err('Unable to insert "Unknown" mining pool');
}
connection.release();
}
}
export default new PoolsParser();

120
backend/src/api/statistics.ts
View File

@@ -53,12 +53,16 @@ class Statistics {
memPoolArray = memPoolArray.filter((tx) => tx.effectiveFeePerVsize);
if (!memPoolArray.length) {
const insertIdZeroed = await this.$createZeroedStatistic();
if (this.newStatisticsEntryCallback && insertIdZeroed) {
const newStats = await this.$get(insertIdZeroed);
if (newStats) {
this.newStatisticsEntryCallback(newStats);
try {
const insertIdZeroed = await this.$createZeroedStatistic();
if (this.newStatisticsEntryCallback && insertIdZeroed) {
const newStats = await this.$get(insertIdZeroed);
if (newStats) {
this.newStatisticsEntryCallback(newStats);
}
}
} catch (e) {
logger.err('Unable to insert zeroed statistics. ' + e);
}
return;
}
@@ -90,59 +94,63 @@ class Statistics {
}
});
const insertId = await this.$create({
added: 'NOW()',
unconfirmed_transactions: memPoolArray.length,
tx_per_second: txPerSecond,
vbytes_per_second: Math.round(vBytesPerSecond),
mempool_byte_weight: totalWeight,
total_fee: totalFee,
fee_data: '',
vsize_1: weightVsizeFees['1'] || 0,
vsize_2: weightVsizeFees['2'] || 0,
vsize_3: weightVsizeFees['3'] || 0,
vsize_4: weightVsizeFees['4'] || 0,
vsize_5: weightVsizeFees['5'] || 0,
vsize_6: weightVsizeFees['6'] || 0,
vsize_8: weightVsizeFees['8'] || 0,
vsize_10: weightVsizeFees['10'] || 0,
vsize_12: weightVsizeFees['12'] || 0,
vsize_15: weightVsizeFees['15'] || 0,
vsize_20: weightVsizeFees['20'] || 0,
vsize_30: weightVsizeFees['30'] || 0,
vsize_40: weightVsizeFees['40'] || 0,
vsize_50: weightVsizeFees['50'] || 0,
vsize_60: weightVsizeFees['60'] || 0,
vsize_70: weightVsizeFees['70'] || 0,
vsize_80: weightVsizeFees['80'] || 0,
vsize_90: weightVsizeFees['90'] || 0,
vsize_100: weightVsizeFees['100'] || 0,
vsize_125: weightVsizeFees['125'] || 0,
vsize_150: weightVsizeFees['150'] || 0,
vsize_175: weightVsizeFees['175'] || 0,
vsize_200: weightVsizeFees['200'] || 0,
vsize_250: weightVsizeFees['250'] || 0,
vsize_300: weightVsizeFees['300'] || 0,
vsize_350: weightVsizeFees['350'] || 0,
vsize_400: weightVsizeFees['400'] || 0,
vsize_500: weightVsizeFees['500'] || 0,
vsize_600: weightVsizeFees['600'] || 0,
vsize_700: weightVsizeFees['700'] || 0,
vsize_800: weightVsizeFees['800'] || 0,
vsize_900: weightVsizeFees['900'] || 0,
vsize_1000: weightVsizeFees['1000'] || 0,
vsize_1200: weightVsizeFees['1200'] || 0,
vsize_1400: weightVsizeFees['1400'] || 0,
vsize_1600: weightVsizeFees['1600'] || 0,
vsize_1800: weightVsizeFees['1800'] || 0,
vsize_2000: weightVsizeFees['2000'] || 0,
});
try {
const insertId = await this.$create({
added: 'NOW()',
unconfirmed_transactions: memPoolArray.length,
tx_per_second: txPerSecond,
vbytes_per_second: Math.round(vBytesPerSecond),
mempool_byte_weight: totalWeight,
total_fee: totalFee,
fee_data: '',
vsize_1: weightVsizeFees['1'] || 0,
vsize_2: weightVsizeFees['2'] || 0,
vsize_3: weightVsizeFees['3'] || 0,
vsize_4: weightVsizeFees['4'] || 0,
vsize_5: weightVsizeFees['5'] || 0,
vsize_6: weightVsizeFees['6'] || 0,
vsize_8: weightVsizeFees['8'] || 0,
vsize_10: weightVsizeFees['10'] || 0,
vsize_12: weightVsizeFees['12'] || 0,
vsize_15: weightVsizeFees['15'] || 0,
vsize_20: weightVsizeFees['20'] || 0,
vsize_30: weightVsizeFees['30'] || 0,
vsize_40: weightVsizeFees['40'] || 0,
vsize_50: weightVsizeFees['50'] || 0,
vsize_60: weightVsizeFees['60'] || 0,
vsize_70: weightVsizeFees['70'] || 0,
vsize_80: weightVsizeFees['80'] || 0,
vsize_90: weightVsizeFees['90'] || 0,
vsize_100: weightVsizeFees['100'] || 0,
vsize_125: weightVsizeFees['125'] || 0,
vsize_150: weightVsizeFees['150'] || 0,
vsize_175: weightVsizeFees['175'] || 0,
vsize_200: weightVsizeFees['200'] || 0,
vsize_250: weightVsizeFees['250'] || 0,
vsize_300: weightVsizeFees['300'] || 0,
vsize_350: weightVsizeFees['350'] || 0,
vsize_400: weightVsizeFees['400'] || 0,
vsize_500: weightVsizeFees['500'] || 0,
vsize_600: weightVsizeFees['600'] || 0,
vsize_700: weightVsizeFees['700'] || 0,
vsize_800: weightVsizeFees['800'] || 0,
vsize_900: weightVsizeFees['900'] || 0,
vsize_1000: weightVsizeFees['1000'] || 0,
vsize_1200: weightVsizeFees['1200'] || 0,
vsize_1400: weightVsizeFees['1400'] || 0,
vsize_1600: weightVsizeFees['1600'] || 0,
vsize_1800: weightVsizeFees['1800'] || 0,
vsize_2000: weightVsizeFees['2000'] || 0,
});
if (this.newStatisticsEntryCallback && insertId) {
const newStats = await this.$get(insertId);
if (newStats) {
this.newStatisticsEntryCallback(newStats);
if (this.newStatisticsEntryCallback && insertId) {
const newStats = await this.$get(insertId);
if (newStats) {
this.newStatisticsEntryCallback(newStats);
}
}
} catch (e) {
logger.err('Unable to insert statistics. ' + e);
}
}

8
backend/src/api/transaction-utils.ts
View File

@@ -44,6 +44,14 @@ class TransactionUtils {
}
return transactionExtended;
}
public hex2ascii(hex: string) {
let str = '';
for (let i = 0; i < hex.length; i += 2) {
str += String.fromCharCode(parseInt(hex.substr(i, 2), 16));
}
return str;
}
}
export default new TransactionUtils();

6
backend/src/config.ts
View File

@@ -14,6 +14,7 @@ interface IConfig {
BLOCK_WEIGHT_UNITS: number;
INITIAL_BLOCKS_AMOUNT: number;
MEMPOOL_BLOCKS_AMOUNT: number;
INDEXING_BLOCKS_AMOUNT: number;
PRICE_FEED_UPDATE_INTERVAL: number;
USE_SECOND_NODE_FOR_MINFEE: boolean;
EXTERNAL_ASSETS: string[];
@@ -77,9 +78,12 @@ const defaults: IConfig = {
'BLOCK_WEIGHT_UNITS': 4000000,
'INITIAL_BLOCKS_AMOUNT': 8,
'MEMPOOL_BLOCKS_AMOUNT': 8,
'INDEXING_BLOCKS_AMOUNT': 1100, // 0 = disable indexing, -1 = index all blocks
'PRICE_FEED_UPDATE_INTERVAL': 3600,
'USE_SECOND_NODE_FOR_MINFEE': false,
'EXTERNAL_ASSETS': [],
'EXTERNAL_ASSETS': [
'https://mempool.space/resources/pools.json'
]
},
'ESPLORA': {
'REST_API_URL': 'http://127.0.0.1:3000',

14
backend/src/index.ts
View File

@@ -22,6 +22,7 @@ import loadingIndicators from './api/loading-indicators';
import mempool from './api/mempool';
import elementsParser from './api/liquid/elements-parser';
import databaseMigration from './api/database-migration';
import poolsParser from './api/pools-parser';
import syncAssets from './sync-assets';
import icons from './api/liquid/icons';
import { Common } from './api/common';
@@ -88,6 +89,7 @@ class Server {
await checkDbConnection();
try {
await databaseMigration.$initializeOrMigrateDatabase();
await poolsParser.migratePoolsJson();
} catch (e) {
throw new Error(e instanceof Error ? e.message : 'Error');
}
@@ -136,6 +138,8 @@ class Server {
}
await blocks.$updateBlocks();
await memPool.$updateMempool();
blocks.$generateBlockDatabase();
setTimeout(this.runMainUpdateLoop.bind(this), config.MEMPOOL.POLL_RATE_MS);
this.currentBackendRetryInterval = 5;
} catch (e) {
@@ -252,6 +256,16 @@ class Server {
.get(config.MEMPOOL.API_URL_PREFIX + 'statistics/1y', routes.$getStatisticsByTime.bind(routes, '1y'))
.get(config.MEMPOOL.API_URL_PREFIX + 'statistics/2y', routes.$getStatisticsByTime.bind(routes, '2y'))
.get(config.MEMPOOL.API_URL_PREFIX + 'statistics/3y', routes.$getStatisticsByTime.bind(routes, '3y'))
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/pools/24h', routes.$getPools.bind(routes, '24h'))
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/pools/3d', routes.$getPools.bind(routes, '3d'))
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/pools/1w', routes.$getPools.bind(routes, '1w'))
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/pools/1m', routes.$getPools.bind(routes, '1m'))
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/pools/3m', routes.$getPools.bind(routes, '3m'))
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/pools/6m', routes.$getPools.bind(routes, '6m'))
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/pools/1y', routes.$getPools.bind(routes, '1y'))
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/pools/2y', routes.$getPools.bind(routes, '2y'))
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/pools/3y', routes.$getPools.bind(routes, '3y'))
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/pools/all', routes.$getPools.bind(routes, 'all'))
;
}

20
backend/src/mempool.interfaces.ts
View File

@@ -1,5 +1,25 @@
import { IEsploraApi } from './api/bitcoin/esplora-api.interface';
export interface PoolTag {
id: number | null, // mysql row id
name: string,
link: string,
regexes: string, // JSON array
addresses: string, // JSON array
}
export interface PoolInfo {
poolId: number, // mysql row id
name: string,
link: string,
blockCount: number,
}
export interface PoolStats extends PoolInfo {
rank: number,
emptyBlocks: number,
}
export interface MempoolBlock {
blockSize: number;
blockVSize: number;

128
backend/src/repositories/BlocksRepository.ts Normal file
View File

@@ -0,0 +1,128 @@
import { BlockExtended, PoolTag } from '../mempool.interfaces';
import { DB } from '../database';
import logger from '../logger';
export interface EmptyBlocks {
emptyBlocks: number;
poolId: number;
}
class BlocksRepository {
/**
* Save indexed block data in the database
*/
public async $saveBlockInDatabase(
block: BlockExtended,
blockHash: string,
coinbaseHex: string | undefined,
poolTag: PoolTag
) {
const connection = await DB.pool.getConnection();
try {
const query = `INSERT INTO blocks(
height, hash, blockTimestamp, size,
weight, tx_count, coinbase_raw, difficulty,
pool_id, fees, fee_span, median_fee
) VALUE (
?, ?, FROM_UNIXTIME(?), ?,
?, ?, ?, ?,
?, ?, ?, ?
)`;
const params: any[] = [
block.height, blockHash, block.timestamp, block.size,
block.weight, block.tx_count, coinbaseHex ? coinbaseHex : '', block.difficulty,
poolTag.id, 0, '[]', block.medianFee,
];
await connection.query(query, params);
} catch (e) {
logger.err('$saveBlockInDatabase() error' + (e instanceof Error ? e.message : e));
}
connection.release();
}
/**
* Get all block height that have not been indexed between [startHeight, endHeight]
*/
public async $getMissingBlocksBetweenHeights(startHeight: number, endHeight: number): Promise<number[]> {
if (startHeight < endHeight) {
return [];
}
const connection = await DB.pool.getConnection();
const [rows] : any[] = await connection.query(`
SELECT height
FROM blocks
WHERE height <= ${startHeight} AND height >= ${endHeight}
ORDER BY height DESC;
`);
connection.release();
const indexedBlockHeights: number[] = [];
rows.forEach((row: any) => { indexedBlockHeights.push(row.height); });
const seekedBlocks: number[] = Array.from(Array(startHeight - endHeight + 1).keys(), n => n + endHeight).reverse();
const missingBlocksHeights = seekedBlocks.filter(x => indexedBlockHeights.indexOf(x) === -1);
return missingBlocksHeights;
}
/**
* Count empty blocks for all pools
*/
public async $countEmptyBlocks(interval: string | null): Promise<EmptyBlocks[]> {
const query = `
SELECT pool_id as poolId
FROM blocks
WHERE tx_count = 1` +
(interval != null ? ` AND blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()` : ``)
;
const connection = await DB.pool.getConnection();
const [rows] = await connection.query(query);
connection.release();
return <EmptyBlocks[]>rows;
}
/**
* Get blocks count for a period
*/
public async $blockCount(interval: string | null): Promise<number> {
const query = `
SELECT count(height) as blockCount
FROM blocks` +
(interval != null ? ` WHERE blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()` : ``)
;
const connection = await DB.pool.getConnection();
const [rows] = await connection.query(query);
connection.release();
return <number>rows[0].blockCount;
}
/**
* Get the oldest indexed block
*/
public async $oldestBlockTimestamp(): Promise<number> {
const connection = await DB.pool.getConnection();
const [rows]: any[] = await connection.query(`
SELECT blockTimestamp
FROM blocks
ORDER BY height
LIMIT 1;
`);
connection.release();
if (rows.length <= 0) {
return -1;
}
return <number>rows[0].blockTimestamp;
}
}
export default new BlocksRepository();

46
backend/src/repositories/PoolsRepository.ts Normal file
View File

@@ -0,0 +1,46 @@
import { DB } from '../database';
import { PoolInfo, PoolTag } from '../mempool.interfaces';
class PoolsRepository {
/**
* Get all pools tagging info
*/
public async $getPools(): Promise<PoolTag[]> {
const connection = await DB.pool.getConnection();
const [rows] = await connection.query('SELECT * FROM pools;');
connection.release();
return <PoolTag[]>rows;
}
/**
* Get unknown pool tagging info
*/
public async $getUnknownPool(): Promise<PoolTag> {
const connection = await DB.pool.getConnection();
const [rows] = await connection.query('SELECT * FROM pools where name = "Unknown"');
connection.release();
return <PoolTag>rows[0];
}
/**
* Get basic pool info and block count
*/
public async $getPoolsInfo(interval: string | null): Promise<PoolInfo[]> {
const query = `
SELECT COUNT(height) as blockCount, pool_id as poolId, pools.name as name, pools.link as link
FROM blocks
JOIN pools on pools.id = pool_id` +
(interval != null ? ` WHERE blocks.blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()` : ``) +
` GROUP BY pool_id
ORDER BY COUNT(height) DESC
`;
const connection = await DB.pool.getConnection();
const [rows] = await connection.query(query);
connection.release();
return <PoolInfo[]>rows;
}
}
export default new PoolsRepository();

13
backend/src/routes.ts
View File

@@ -20,6 +20,7 @@ import { Common } from './api/common';
import bitcoinClient from './api/bitcoin/bitcoin-client';
import elementsParser from './api/liquid/elements-parser';
import icons from './api/liquid/icons';
import miningStats from './api/mining';
class Routes {
constructor() {}
@@ -531,6 +532,18 @@ class Routes {
}
}
public async $getPools(interval: string, req: Request, res: Response) {
try {
let stats = await miningStats.$getPoolsStats(interval);
res.header('Pragma', 'public');
res.header('Cache-control', 'public');
res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
res.json(stats);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
}
}
public async getBlock(req: Request, res: Response) {
try {
const result = await bitcoinApi.$getBlock(req.params.hash);