machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into nymkappa/bugfix/bisq-dump-file

Browse Source
This commit is contained in:
wiz
2022-06-01 00:14:43 +09:00
committed by GitHub
parent 7e8e4b1e6c 6a3df95d4c
commit 72c3eea863
150 changed files with 50932 additions and 50074 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts
View File

@@ -2,7 +2,7 @@ import { IEsploraApi } from './esplora-api.interface';
export interface AbstractBitcoinApi {
$getRawMempool(): Promise<IEsploraApi.Transaction['txid'][]>;
$getRawTransaction(txId: string, skipConversion?: boolean, addPrevout?: boolean, blockHash?: string): Promise<IEsploraApi.Transaction>;
$getRawTransaction(txId: string, skipConversion?: boolean, addPrevout?: boolean, lazyPrevouts?: boolean): Promise<IEsploraApi.Transaction>;
$getBlockHeightTip(): Promise<number>;
$getTxIdsForBlock(hash: string): Promise<string[]>;
$getBlockHash(height: number): Promise<string>;

82
backend/src/api/bitcoin/bitcoin-api.ts
View File

@@ -14,14 +14,32 @@ class BitcoinApi implements AbstractBitcoinApi {
this.bitcoindClient = bitcoinClient;
}
$getRawTransaction(txId: string, skipConversion = false, addPrevout = false, blockHash?: string): Promise<IEsploraApi.Transaction> {
static convertBlock(block: IBitcoinApi.Block): IEsploraApi.Block {
return {
id: block.hash,
height: block.height,
version: block.version,
timestamp: block.time,
bits: parseInt(block.bits, 16),
nonce: block.nonce,
difficulty: block.difficulty,
merkle_root: block.merkleroot,
tx_count: block.nTx,
size: block.size,
weight: block.weight,
previousblockhash: block.previousblockhash,
};
}
$getRawTransaction(txId: string, skipConversion = false, addPrevout = false, lazyPrevouts = false): Promise<IEsploraApi.Transaction> {
// If the transaction is in the mempool we already converted and fetched the fee. Only prevouts are missing
const txInMempool = mempool.getMempool()[txId];
if (txInMempool && addPrevout) {
return this.$addPrevouts(txInMempool);
}
return this.bitcoindClient.getRawTransaction(txId, true, blockHash)
return this.bitcoindClient.getRawTransaction(txId, true)
.then((transaction: IBitcoinApi.Transaction) => {
if (skipConversion) {
transaction.vout.forEach((vout) => {
@@ -29,7 +47,7 @@ class BitcoinApi implements AbstractBitcoinApi {
});
return transaction;
}
return this.$convertTransaction(transaction, addPrevout);
return this.$convertTransaction(transaction, addPrevout, lazyPrevouts);
})
.catch((e: Error) => {
if (e.message.startsWith('The genesis block coinbase')) {
@@ -109,7 +127,7 @@ class BitcoinApi implements AbstractBitcoinApi {
const outSpends: IEsploraApi.Outspend[] = [];
const tx = await this.$getRawTransaction(txId, true, false);
for (let i = 0; i < tx.vout.length; i++) {
if (tx.status && tx.status.block_height == 0) {
if (tx.status && tx.status.block_height === 0) {
outSpends.push({
spent: false
});
@@ -128,7 +146,7 @@ class BitcoinApi implements AbstractBitcoinApi {
return this.bitcoindClient.getNetworkHashPs(120, blockHeight);
}
protected async $convertTransaction(transaction: IBitcoinApi.Transaction, addPrevout: boolean): Promise<IEsploraApi.Transaction> {
protected async $convertTransaction(transaction: IBitcoinApi.Transaction, addPrevout: boolean, lazyPrevouts = false): Promise<IEsploraApi.Transaction> {
let esploraTransaction: IEsploraApi.Transaction = {
txid: transaction.txid,
version: transaction.version,
@@ -174,35 +192,15 @@ class BitcoinApi implements AbstractBitcoinApi {
};
}
if (transaction.confirmations) {
esploraTransaction = await this.$calculateFeeFromInputs(esploraTransaction, addPrevout);
} else {
if (addPrevout) {
esploraTransaction = await this.$calculateFeeFromInputs(esploraTransaction, false, lazyPrevouts);
} else if (!transaction.confirmations) {
esploraTransaction = await this.$appendMempoolFeeData(esploraTransaction);
if (addPrevout) {
esploraTransaction = await this.$calculateFeeFromInputs(esploraTransaction, addPrevout);
}
}
return esploraTransaction;
}
static convertBlock(block: IBitcoinApi.Block): IEsploraApi.Block {
return {
id: block.hash,
height: block.height,
version: block.version,
timestamp: block.time,
bits: parseInt(block.bits, 16),
nonce: block.nonce,
difficulty: block.difficulty,
merkle_root: block.merkleroot,
tx_count: block.nTx,
size: block.size,
weight: block.weight,
previousblockhash: block.previousblockhash,
};
}
private translateScriptPubKeyType(outputType: string): string {
const map = {
'pubkey': 'p2pk',
@@ -245,7 +243,7 @@ class BitcoinApi implements AbstractBitcoinApi {
if (vin.prevout) {
continue;
}
const innerTx = await this.$getRawTransaction(vin.txid, false);
const innerTx = await this.$getRawTransaction(vin.txid, false, false);
vin.prevout = innerTx.vout[vin.vout];
this.addInnerScriptsToVin(vin);
}
@@ -271,22 +269,30 @@ class BitcoinApi implements AbstractBitcoinApi {
return this.bitcoindClient.getRawMemPool(true);
}
private async $calculateFeeFromInputs(transaction: IEsploraApi.Transaction, addPrevout: boolean): Promise<IEsploraApi.Transaction> {
private async $calculateFeeFromInputs(transaction: IEsploraApi.Transaction, addPrevout: boolean, lazyPrevouts: boolean): Promise<IEsploraApi.Transaction> {
if (transaction.vin[0].is_coinbase) {
transaction.fee = 0;
return transaction;
}
let totalIn = 0;
for (const vin of transaction.vin) {
const innerTx = await this.$getRawTransaction(vin.txid, !addPrevout);
if (addPrevout) {
vin.prevout = innerTx.vout[vin.vout];
this.addInnerScriptsToVin(vin);
for (let i = 0; i < transaction.vin.length; i++) {
if (lazyPrevouts && i > 12) {
transaction.vin[i].lazy = true;
continue;
}
totalIn += innerTx.vout[vin.vout].value;
const innerTx = await this.$getRawTransaction(transaction.vin[i].txid, false, false);
transaction.vin[i].prevout = innerTx.vout[transaction.vin[i].vout];
this.addInnerScriptsToVin(transaction.vin[i]);
totalIn += innerTx.vout[transaction.vin[i].vout].value;
}
if (lazyPrevouts && transaction.vin.length > 12) {
transaction.fee = -1;
} else {
const totalOut = transaction.vout.reduce((p, output) => p + output.value, 0);
transaction.fee = parseFloat((totalIn - totalOut).toFixed(8));
}
const totalOut = transaction.vout.reduce((p, output) => p + output.value, 0);
transaction.fee = parseFloat((totalIn - totalOut).toFixed(8));
return transaction;
}

2
backend/src/api/bitcoin/esplora-api.interface.ts
View File

@@ -33,6 +33,8 @@ export namespace IEsploraApi {
// Elements
is_pegin?: boolean;
issuance?: Issuance;
// Custom
lazy?: boolean;
}
interface Issuance {

91
backend/src/api/blocks.ts
View File

@@ -15,6 +15,9 @@ import BitcoinApi from './bitcoin/bitcoin-api';
import { prepareBlock } from '../utils/blocks-utils';
import BlocksRepository from '../repositories/BlocksRepository';
import HashratesRepository from '../repositories/HashratesRepository';
import indexer from '../indexer';
import fiatConversion from './fiat-conversion';
import RatesRepository from '../repositories/RatesRepository';
class Blocks {
private blocks: BlockExtended[] = [];
@@ -23,9 +26,6 @@ class Blocks {
private lastDifficultyAdjustmentTime = 0;
private previousDifficultyRetarget = 0;
private newBlockCallbacks: ((block: BlockExtended, txIds: string[], transactions: TransactionExtended[]) => void)[] = [];
private blockIndexingStarted = false;
public blockIndexingCompleted = false;
public reindexFlag = false;
constructor() { }
@@ -134,7 +134,7 @@ class Blocks {
blockExtended.extras.avgFeeRate = stats.avgfeerate;
}
if (Common.indexingEnabled()) {
if (['mainnet', 'testnet', 'signet', 'regtest'].includes(config.MEMPOOL.NETWORK)) {
let pool: PoolTag;
if (blockExtended.extras?.coinbaseTx !== undefined) {
pool = await this.$findBlockMiner(blockExtended.extras?.coinbaseTx);
@@ -143,7 +143,8 @@ class Blocks {
}
if (!pool) { // We should never have this situation in practise
logger.warn(`Cannot assign pool to block ${blockExtended.height} and 'unknown' pool does not exist. Check your "pools" table entries`);
logger.warn(`Cannot assign pool to block ${blockExtended.height} and 'unknown' pool does not exist. ` +
`Check your "pools" table entries`);
return blockExtended;
}
@@ -196,24 +197,15 @@ class Blocks {
* [INDEXING] Index all blocks metadata for the mining dashboard
*/
public async $generateBlockDatabase() {
if (this.blockIndexingStarted && !this.reindexFlag) {
return;
}
this.reindexFlag = false;
const blockchainInfo = await bitcoinClient.getBlockchainInfo();
if (blockchainInfo.blocks !== blockchainInfo.headers) { // Wait for node to sync
return;
}
this.blockIndexingStarted = true;
this.blockIndexingCompleted = false;
try {
let currentBlockHeight = blockchainInfo.blocks;
let indexingBlockAmount = config.MEMPOOL.INDEXING_BLOCKS_AMOUNT;
let indexingBlockAmount = Math.min(config.MEMPOOL.INDEXING_BLOCKS_AMOUNT, blockchainInfo.blocks);
if (indexingBlockAmount <= -1) {
indexingBlockAmount = currentBlockHeight + 1;
}
@@ -274,14 +266,14 @@ class Blocks {
loadingIndicators.setProgress('block-indexing', 100);
} catch (e) {
logger.err('Block indexing failed. Trying again later. Reason: ' + (e instanceof Error ? e.message : e));
this.blockIndexingStarted = false;
loadingIndicators.setProgress('block-indexing', 100);
return;
}
const chainValid = await BlocksRepository.$validateChain();
this.reindexFlag = !chainValid;
this.blockIndexingCompleted = chainValid;
if (!chainValid) {
indexer.reindex();
}
}
public async $updateBlocks() {
@@ -298,6 +290,8 @@ class Blocks {
logger.info(`${blockHeightTip - this.currentBlockHeight} blocks since tip. Fast forwarding to the ${config.MEMPOOL.INITIAL_BLOCKS_AMOUNT} recent blocks`);
this.currentBlockHeight = blockHeightTip - config.MEMPOOL.INITIAL_BLOCKS_AMOUNT;
fastForwarded = true;
logger.info(`Re-indexing skipped blocks and corresponding hashrates data`);
indexer.reindex(); // Make sure to index the skipped blocks #1619
}
if (!this.lastDifficultyAdjustmentTime) {
@@ -349,6 +343,9 @@ class Blocks {
await blocksRepository.$saveBlockInDatabase(blockExtended);
}
}
if (fiatConversion.ratesInitialized === true) {
await RatesRepository.$saveRate(blockExtended.height, fiatConversion.getConversionRates());
}
if (block.height % 2016 === 0) {
this.previousDifficultyRetarget = (block.difficulty - this.currentDifficulty) / this.currentDifficulty * 100;
@@ -389,10 +386,43 @@ class Blocks {
return prepareBlock(blockExtended);
}
public async $getBlocksExtras(fromHeight?: number, limit: number = 15): Promise<BlockExtended[]> {
// Note - This API is breaking if indexing is not available. For now it is okay because we only
// use it for the mining pages, and mining pages should not be available if indexing is turned off.
// I'll need to fix it before we refactor the block(s) related pages
/**
* Index a block by hash if it's missing from the database. Returns the block after indexing
*/
public async $getBlock(hash: string): Promise<BlockExtended | IEsploraApi.Block> {
// Check the memory cache
const blockByHash = this.getBlocks().find((b) => b.id === hash);
if (blockByHash) {
return blockByHash;
}
// Block has already been indexed
if (Common.indexingEnabled()) {
const dbBlock = await blocksRepository.$getBlockByHash(hash);
if (dbBlock != null) {
return prepareBlock(dbBlock);
}
}
const block = await bitcoinApi.$getBlock(hash);
// Not Bitcoin network, return the block as it
if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) {
return block;
}
// Bitcoin network, add our custom data on top
const transactions = await this.$getTransactionsExtended(hash, block.height, true);
const blockExtended = await this.$getBlockExtended(block, transactions);
if (Common.indexingEnabled()) {
delete(blockExtended['coinbaseTx']);
await blocksRepository.$saveBlockInDatabase(blockExtended);
}
return blockExtended;
}
public async $getBlocks(fromHeight?: number, limit: number = 15): Promise<BlockExtended[]> {
try {
let currentHeight = fromHeight !== undefined ? fromHeight : this.getCurrentBlockHeight();
const returnBlocks: BlockExtended[] = [];
@@ -401,25 +431,32 @@ class Blocks {
return returnBlocks;
}
if (currentHeight === 0 && Common.indexingEnabled()) {
currentHeight = await blocksRepository.$mostRecentBlockHeight();
}
// Check if block height exist in local cache to skip the hash lookup
const blockByHeight = this.getBlocks().find((b) => b.height === currentHeight);
let startFromHash: string | null = null;
if (blockByHeight) {
startFromHash = blockByHeight.id;
} else {
} else if (!Common.indexingEnabled()) {
startFromHash = await bitcoinApi.$getBlockHash(currentHeight);
}
let nextHash = startFromHash;
for (let i = 0; i < limit && currentHeight >= 0; i++) {
let block = this.getBlocks().find((b) => b.height === currentHeight);
if (!block && Common.indexingEnabled()) {
if (block) {
returnBlocks.push(block);
} else if (Common.indexingEnabled()) {
block = await this.$indexBlock(currentHeight);
} else if (!block) {
returnBlocks.push(block);
} else if (nextHash != null) {
block = prepareBlock(await bitcoinApi.$getBlock(nextHash));
nextHash = block.previousblockhash;
returnBlocks.push(block);
}
returnBlocks.push(block);
nextHash = block.previousblockhash;
currentHeight--;
}

6
backend/src/api/common.ts
View File

@@ -169,12 +169,12 @@ export class Common {
default: return null;
}
}
static indexingEnabled(): boolean {
return (
['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) &&
['mainnet', 'testnet', 'signet', 'regtest'].includes(config.MEMPOOL.NETWORK) &&
config.DATABASE.ENABLED === true &&
config.MEMPOOL.INDEXING_BLOCKS_AMOUNT != 0
config.MEMPOOL.INDEXING_BLOCKS_AMOUNT !== 0
);
}
}

18
backend/src/api/database-migration.ts
View File

@@ -4,7 +4,7 @@ import logger from '../logger';
import { Common } from './common';
class DatabaseMigration {
private static currentVersion = 17;
private static currentVersion = 19;
private queryTimeout = 120000;
private statisticsAddedIndexed = false;
@@ -180,6 +180,14 @@ class DatabaseMigration {
if (databaseSchemaVersion < 17 && isBitcoin === true) {
await this.$executeQuery('ALTER TABLE `pools` ADD `slug` CHAR(50) NULL');
}
if (databaseSchemaVersion < 18 && isBitcoin === true) {
await this.$executeQuery('ALTER TABLE `blocks` ADD INDEX `hash` (`hash`);');
}
if (databaseSchemaVersion < 19) {
await this.$executeQuery(this.getCreateRatesTableQuery(), await this.$checkIfTableExists('rates'));
}
} catch (e) {
throw e;
}
@@ -462,6 +470,14 @@ class DatabaseMigration {
) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
}
private getCreateRatesTableQuery(): string {
return `CREATE TABLE IF NOT EXISTS rates (
height int(10) unsigned NOT NULL,
bisq_rates JSON NOT NULL,
PRIMARY KEY (height)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
}
public async $truncateIndexedData(tables: string[]) {
const allowedTables = ['blocks', 'hashrates'];

2
backend/src/api/difficulty-adjustment.ts
View File

@@ -32,7 +32,7 @@ class DifficultyAdjustmentApi {
}
}
let timeAvgMins = blocksInEpoch ? diff / blocksInEpoch / 60 : 10;
let timeAvgMins = blocksInEpoch && blocksInEpoch > 146 ? diff / blocksInEpoch / 60 : 10;
// Testnet difficulty is set to 1 after 20 minutes of no blocks,
// therefore the time between blocks will always be below 20 minutes (1200s).

3
backend/src/api/fee-api.ts
View File

@@ -1,4 +1,3 @@
import config from '../config';
import { MempoolBlock } from '../mempool.interfaces';
import { Common } from './common';
import mempool from './mempool';
@@ -19,6 +18,7 @@ class FeeApi {
'fastestFee': this.defaultFee,
'halfHourFee': this.defaultFee,
'hourFee': this.defaultFee,
'economyFee': minimumFee,
'minimumFee': minimumFee,
};
}
@@ -31,6 +31,7 @@ class FeeApi {
'fastestFee': firstMedianFee,
'halfHourFee': secondMedianFee,
'hourFee': thirdMedianFee,
'economyFee': Math.min(2 * minimumFee, thirdMedianFee),
'minimumFee': minimumFee,
};
}

28
backend/src/api/fiat-conversion.ts
View File

@@ -6,12 +6,19 @@ import backendInfo from './backend-info';
import { SocksProxyAgent } from 'socks-proxy-agent';
class FiatConversion {
private conversionRates: IConversionRates = {
'USD': 0
};
private debasingFiatCurrencies = ['AED', 'AUD', 'BDT', 'BHD', 'BMD', 'BRL', 'CAD', 'CHF', 'CLP',
'CNY', 'CZK', 'DKK', 'EUR', 'GBP', 'HKD', 'HUF', 'IDR', 'ILS', 'INR', 'JPY', 'KRW', 'KWD',
'LKR', 'MMK', 'MXN', 'MYR', 'NGN', 'NOK', 'NZD', 'PHP', 'PKR', 'PLN', 'RUB', 'SAR', 'SEK',
'SGD', 'THB', 'TRY', 'TWD', 'UAH', 'USD', 'VND', 'ZAR'];
private conversionRates: IConversionRates = {};
private ratesChangedCallback: ((rates: IConversionRates) => void) | undefined;
public ratesInitialized = false; // If true, it means rates are ready for use
constructor() { }
constructor() {
for (const fiat of this.debasingFiatCurrencies) {
this.conversionRates[fiat] = 0;
}
}
public setProgressChangedCallback(fn: (rates: IConversionRates) => void) {
this.ratesChangedCallback = fn;
@@ -62,13 +69,14 @@ class FiatConversion {
response = await axios.get(fiatConversionUrl, { headers: headers, timeout: 10000 });
}
const usd = response.data.data.find((item: any) => item.currencyCode === 'USD');
for (const rate of response.data.data) {
if (this.debasingFiatCurrencies.includes(rate.currencyCode) && rate.provider === 'Bisq-Aggregate') {
this.conversionRates[rate.currencyCode] = Math.round(100 * rate.price) / 100;
}
}
this.conversionRates = {
'USD': usd.price,
};
logger.debug(`USD Conversion Rate: ${usd.price}`);
this.ratesInitialized = true;
logger.debug(`USD Conversion Rate: ${this.conversionRates.USD}`);
if (this.ratesChangedCallback) {
this.ratesChangedCallback(this.conversionRates);

47
backend/src/api/mining.ts
View File

@@ -4,14 +4,11 @@ import PoolsRepository from '../repositories/PoolsRepository';
import HashratesRepository from '../repositories/HashratesRepository';
import bitcoinClient from './bitcoin/bitcoin-client';
import logger from '../logger';
import blocks from './blocks';
import { Common } from './common';
import loadingIndicators from './loading-indicators';
import { escape } from 'mysql2';
class Mining {
hashrateIndexingStarted = false;
weeklyHashrateIndexingStarted = false;
constructor() {
}
@@ -92,14 +89,18 @@ class Mining {
});
poolsStatistics['pools'] = poolsStats;
poolsStatistics['oldestIndexedBlockTimestamp'] = await BlocksRepository.$oldestBlockTimestamp();
const blockCount: number = await BlocksRepository.$blockCount(null, interval);
poolsStatistics['blockCount'] = blockCount;
const totalBlock24h: number = await BlocksRepository.$blockCount(null, '24h');
const lastBlockHashrate = await bitcoinClient.getNetworkHashPs(totalBlock24h);
poolsStatistics['lastEstimatedHashrate'] = lastBlockHashrate;
try {
poolsStatistics['lastEstimatedHashrate'] = await bitcoinClient.getNetworkHashPs(totalBlock24h);
} catch (e) {
poolsStatistics['lastEstimatedHashrate'] = 0;
logger.debug('Bitcoin Core is not available, using zeroed value for current hashrate');
}
return poolsStatistics;
}
@@ -110,7 +111,7 @@ class Mining {
public async $getPoolStat(slug: string): Promise<object> {
const pool = await PoolsRepository.$getPool(slug);
if (!pool) {
throw new Error(`This mining pool does not exist`);
throw new Error('This mining pool does not exist ' + escape(slug));
}
const blockCount: number = await BlocksRepository.$blockCount(pool.id);
@@ -122,7 +123,12 @@ class Mining {
const blockCount1w: number = await BlocksRepository.$blockCount(pool.id, '1w');
const totalBlock1w: number = await BlocksRepository.$blockCount(null, '1w');
const currentEstimatedkHashrate = await bitcoinClient.getNetworkHashPs(totalBlock24h);
let currentEstimatedHashrate = 0;
try {
currentEstimatedHashrate = await bitcoinClient.getNetworkHashPs(totalBlock24h);
} catch (e) {
logger.debug('Bitcoin Core is not available, using zeroed value for current hashrate');
}
return {
pool: pool,
@@ -136,7 +142,7 @@ class Mining {
'24h': blockCount24h / totalBlock24h,
'1w': blockCount1w / totalBlock1w,
},
estimatedHashrate: currentEstimatedkHashrate * (blockCount24h / totalBlock24h),
estimatedHashrate: currentEstimatedHashrate * (blockCount24h / totalBlock24h),
reportedHashrate: null,
};
}
@@ -152,14 +158,9 @@ class Mining {
* [INDEXING] Generate weekly mining pool hashrate history
*/
public async $generatePoolHashrateHistory(): Promise<void> {
if (!blocks.blockIndexingCompleted || this.hashrateIndexingStarted || this.weeklyHashrateIndexingStarted) {
return;
}
const now = new Date();
try {
this.weeklyHashrateIndexingStarted = true;
const lastestRunDate = await HashratesRepository.$getLatestRun('last_weekly_hashrates_indexing');
// Run only if:
@@ -167,11 +168,9 @@ class Mining {
// * we started a new week (around Monday midnight)
const runIndexing = lastestRunDate === 0 || now.getUTCDay() === 1 && lastestRunDate !== now.getUTCDate();
if (!runIndexing) {
this.weeklyHashrateIndexingStarted = false;
return;
}
} catch (e) {
this.weeklyHashrateIndexingStarted = false;
throw e;
}
@@ -191,6 +190,7 @@ class Mining {
const startedAt = new Date().getTime() / 1000;
let timer = new Date().getTime() / 1000;
logger.debug(`Indexing weekly mining pool hashrate`);
loadingIndicators.setProgress('weekly-hashrate-indexing', 0);
while (toTimestamp > genesisTimestamp) {
@@ -255,7 +255,6 @@ class Mining {
++indexedThisRun;
++totalIndexed;
}
this.weeklyHashrateIndexingStarted = false;
await HashratesRepository.$setLatestRun('last_weekly_hashrates_indexing', new Date().getUTCDate());
if (newlyIndexed > 0) {
logger.info(`Indexed ${newlyIndexed} pools weekly hashrate`);
@@ -263,7 +262,6 @@ class Mining {
loadingIndicators.setProgress('weekly-hashrate-indexing', 100);
} catch (e) {
loadingIndicators.setProgress('weekly-hashrate-indexing', 100);
this.weeklyHashrateIndexingStarted = false;
throw e;
}
}
@@ -272,22 +270,14 @@ class Mining {
* [INDEXING] Generate daily hashrate data
*/
public async $generateNetworkHashrateHistory(): Promise<void> {
if (!blocks.blockIndexingCompleted || this.hashrateIndexingStarted) {
return;
}
try {
this.hashrateIndexingStarted = true;
// We only run this once a day around midnight
const latestRunDate = await HashratesRepository.$getLatestRun('last_hashrates_indexing');
const now = new Date().getUTCDate();
if (now === latestRunDate) {
this.hashrateIndexingStarted = false;
return;
}
} catch (e) {
this.hashrateIndexingStarted = false;
throw e;
}
@@ -305,6 +295,7 @@ class Mining {
const startedAt = new Date().getTime() / 1000;
let timer = new Date().getTime() / 1000;
logger.debug(`Indexing daily network hashrate`);
loadingIndicators.setProgress('daily-hashrate-indexing', 0);
while (toTimestamp > genesisTimestamp) {
@@ -376,14 +367,12 @@ class Mining {
await HashratesRepository.$saveHashrates(hashrates);
await HashratesRepository.$setLatestRun('last_hashrates_indexing', new Date().getUTCDate());
this.hashrateIndexingStarted = false;
if (newlyIndexed > 0) {
logger.info(`Indexed ${newlyIndexed} day of network hashrate`);
}
loadingIndicators.setProgress('daily-hashrate-indexing', 100);
} catch (e) {
loadingIndicators.setProgress('daily-hashrate-indexing', 100);
this.hashrateIndexingStarted = false;
throw e;
}
}

4
backend/src/api/transaction-utils.ts
View File

@@ -21,8 +21,8 @@ class TransactionUtils {
};
}
public async $getTransactionExtended(txId: string, addPrevouts = false): Promise<TransactionExtended> {
const transaction: IEsploraApi.Transaction = await bitcoinApi.$getRawTransaction(txId, false, addPrevouts);
public async $getTransactionExtended(txId: string, addPrevouts = false, lazyPrevouts = false): Promise<TransactionExtended> {
const transaction: IEsploraApi.Transaction = await bitcoinApi.$getRawTransaction(txId, false, addPrevouts, lazyPrevouts);
return this.extendTransaction(transaction);
}

53
backend/src/index.ts
View File

@@ -25,10 +25,8 @@ import databaseMigration from './api/database-migration';
import syncAssets from './sync-assets';
import icons from './api/liquid/icons';
import { Common } from './api/common';
import mining from './api/mining';
import HashratesRepository from './repositories/HashratesRepository';
import BlocksRepository from './repositories/BlocksRepository';
import poolsUpdater from './tasks/pools-updater';
import indexer from './indexer';
class Server {
private wss: WebSocket.Server | undefined;
@@ -99,7 +97,7 @@ class Server {
}
await databaseMigration.$initializeOrMigrateDatabase();
if (Common.indexingEnabled()) {
await this.$resetHashratesIndexingState();
await indexer.$resetHashratesIndexingState();
}
} catch (e) {
throw new Error(e instanceof Error ? e.message : 'Error');
@@ -154,7 +152,7 @@ class Server {
await poolsUpdater.updatePoolsJson();
await blocks.$updateBlocks();
await memPool.$updateMempool();
this.$runIndexingWhenReady();
indexer.$run();
setTimeout(this.runMainUpdateLoop.bind(this), config.MEMPOOL.POLL_RATE_MS);
this.currentBackendRetryInterval = 5;
@@ -173,29 +171,6 @@ class Server {
}
}
async $resetHashratesIndexingState() {
try {
await HashratesRepository.$setLatestRun('last_hashrates_indexing', 0);
await HashratesRepository.$setLatestRun('last_weekly_hashrates_indexing', 0);
} catch (e) {
logger.err(`Cannot reset hashrate indexing timestamps. Reason: ` + (e instanceof Error ? e.message : e));
}
}
async $runIndexingWhenReady() {
if (!Common.indexingEnabled() || mempool.hasPriority()) {
return;
}
try {
await blocks.$generateBlockDatabase();
await mining.$generateNetworkHashrateHistory();
await mining.$generatePoolHashrateHistory();
} catch (e) {
logger.err(`Indexing failed, trying again later. Reason: ` + (e instanceof Error ? e.message : e));
}
}
setUpWebsocketHandling() {
if (this.wss) {
websocketHandler.setWebsocketServer(this.wss);
@@ -300,24 +275,12 @@ class Server {
if (Common.indexingEnabled()) {
this.app
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/pools/24h', routes.$getPools.bind(routes, '24h'))
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/pools/3d', routes.$getPools.bind(routes, '3d'))
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/pools/1w', routes.$getPools.bind(routes, '1w'))
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/pools/1m', routes.$getPools.bind(routes, '1m'))
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/pools/3m', routes.$getPools.bind(routes, '3m'))
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/pools/6m', routes.$getPools.bind(routes, '6m'))
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/pools/1y', routes.$getPools.bind(routes, '1y'))
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/pools/2y', routes.$getPools.bind(routes, '2y'))
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/pools/3y', routes.$getPools.bind(routes, '3y'))
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/pools/all', routes.$getPools.bind(routes, 'all'))
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/pools/:interval', routes.$getPools)
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/pool/:slug/hashrate', routes.$getPoolHistoricalHashrate)
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/pool/:slug/blocks', routes.$getPoolBlocks)
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/pool/:slug/blocks/:height', routes.$getPoolBlocks)
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/pool/:slug', routes.$getPool)
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/pool/:slug/:interval', routes.$getPool)
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/hashrate/pools', routes.$getPoolsHistoricalHashrate)
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/hashrate/pools/:interval', routes.$getPoolsHistoricalHashrate)
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/hashrate', routes.$getHistoricalHashrate)
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/hashrate/:interval', routes.$getHistoricalHashrate)
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/reward-stats/:blockCount', routes.$getRewardStats)
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/fees/:interval', routes.$getHistoricalBlockFees)
@@ -349,8 +312,9 @@ class Server {
}
this.app
.get(config.MEMPOOL.API_URL_PREFIX + 'blocks-extras', routes.getBlocksExtras)
.get(config.MEMPOOL.API_URL_PREFIX + 'blocks-extras/:height', routes.getBlocksExtras);
.get(config.MEMPOOL.API_URL_PREFIX + 'blocks', routes.getBlocks.bind(routes))
.get(config.MEMPOOL.API_URL_PREFIX + 'blocks/:height', routes.getBlocks.bind(routes))
.get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash', routes.getBlock);
if (config.MEMPOOL.BACKEND !== 'esplora') {
this.app
@@ -362,10 +326,7 @@ class Server {
.get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/hex', routes.getRawTransaction)
.get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/status', routes.getTransactionStatus)
.get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/outspends', routes.getTransactionOutspends)
.get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash', routes.getBlock)
.get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/header', routes.getBlockHeader)
.get(config.MEMPOOL.API_URL_PREFIX + 'blocks', routes.getBlocks)
.get(config.MEMPOOL.API_URL_PREFIX + 'blocks/:height', routes.getBlocks)
.get(config.MEMPOOL.API_URL_PREFIX + 'blocks/tip/height', routes.getBlockTipHeight)
.get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/txs', routes.getBlockTransactions)
.get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/txs/:index', routes.getBlockTransactions)

52
backend/src/indexer.ts Normal file
View File

@@ -0,0 +1,52 @@
import { Common } from './api/common';
import blocks from './api/blocks';
import mempool from './api/mempool';
import mining from './api/mining';
import logger from './logger';
import HashratesRepository from './repositories/HashratesRepository';
class Indexer {
runIndexer = true;
indexerRunning = false;
constructor() {
}
public reindex() {
this.runIndexer = true;
}
public async $run() {
if (!Common.indexingEnabled() || this.runIndexer === false ||
this.indexerRunning === true || mempool.hasPriority()
) {
return;
}
this.runIndexer = false;
this.indexerRunning = true;
try {
await blocks.$generateBlockDatabase();
await this.$resetHashratesIndexingState();
await mining.$generateNetworkHashrateHistory();
await mining.$generatePoolHashrateHistory();
} catch (e) {
this.reindex();
logger.err(`Indexer failed, trying again later. Reason: ` + (e instanceof Error ? e.message : e));
}
this.indexerRunning = false;
}
async $resetHashratesIndexingState() {
try {
await HashratesRepository.$setLatestRun('last_hashrates_indexing', 0);
await HashratesRepository.$setLatestRun('last_weekly_hashrates_indexing', 0);
} catch (e) {
logger.err(`Cannot reset hashrate indexing timestamps. Reason: ` + (e instanceof Error ? e.message : e));
}
}
}
export default new Indexer();

2
backend/src/mempool.interfaces.ts
View File

@@ -81,7 +81,7 @@ export interface TransactionStripped {
export interface BlockExtension {
totalFees?: number;
medianFee?: number; // Actually the median fee rate that we compute ourself
medianFee?: number;
feeRange?: number[];
reward?: number;
coinbaseTx?: TransactionMinerInfo;

158
backend/src/repositories/BlocksRepository.ts
View File

@@ -5,6 +5,7 @@ import { Common } from '../api/common';
import { prepareBlock } from '../utils/blocks-utils';
import PoolsRepository from './PoolsRepository';
import HashratesRepository from './HashratesRepository';
import { escape } from 'mysql2';
class BlocksRepository {
/**
@@ -120,6 +121,19 @@ class BlocksRepository {
}
}
/**
* Return most recent block height
*/
public async $mostRecentBlockHeight(): Promise<number> {
try {
const [row] = await DB.query('SELECT MAX(height) as maxHeight from blocks');
return row[0]['maxHeight'];
} catch (e) {
logger.err(`Cannot count blocks for this pool (using offset). Reason: ` + (e instanceof Error ? e.message : e));
throw e;
}
}
/**
* Get blocks count for a period
*/
@@ -235,12 +249,30 @@ class BlocksRepository {
public async $getBlocksByPool(slug: string, startHeight?: number): Promise<object[]> {
const pool = await PoolsRepository.$getPool(slug);
if (!pool) {
throw new Error(`This mining pool does not exist`);
throw new Error('This mining pool does not exist ' + escape(slug));
}
const params: any[] = [];
let query = ` SELECT *, UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp,
previous_block_hash as previousblockhash
let query = ` SELECT
height,
hash as id,
UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp,
size,
weight,
tx_count,
coinbase_raw,
difficulty,
fees,
fee_span,
median_fee,
reward,
version,
bits,
nonce,
merkle_root,
previous_block_hash as previousblockhash,
avg_fee,
avg_fee_rate
FROM blocks
WHERE pool_id = ?`;
params.push(pool.id);
@@ -273,11 +305,32 @@ class BlocksRepository {
*/
public async $getBlockByHeight(height: number): Promise<object | null> {
try {
const [rows]: any[] = await DB.query(`
SELECT *, UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp,
pools.id as pool_id, pools.name as pool_name, pools.link as pool_link, pools.slug as pool_slug,
pools.addresses as pool_addresses, pools.regexes as pool_regexes,
previous_block_hash as previousblockhash
const [rows]: any[] = await DB.query(`SELECT
height,
hash as id,
UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp,
size,
weight,
tx_count,
coinbase_raw,
difficulty,
pools.id as pool_id,
pools.name as pool_name,
pools.link as pool_link,
pools.slug as pool_slug,
pools.addresses as pool_addresses,
pools.regexes as pool_regexes,
fees,
fee_span,
median_fee,
reward,
version,
bits,
nonce,
merkle_root,
previous_block_hash as previousblockhash,
avg_fee,
avg_fee_rate
FROM blocks
JOIN pools ON blocks.pool_id = pools.id
WHERE height = ${height};
@@ -287,6 +340,7 @@ class BlocksRepository {
return null;
}
rows[0].fee_span = JSON.parse(rows[0].fee_span);
return rows[0];
} catch (e) {
logger.err(`Cannot get indexed block ${height}. Reason: ` + (e instanceof Error ? e.message : e));
@@ -294,6 +348,34 @@ class BlocksRepository {
}
}
/**
* Get one block by hash
*/
public async $getBlockByHash(hash: string): Promise<object | null> {
try {
const query = `
SELECT *, UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp, hash as id,
pools.id as pool_id, pools.name as pool_name, pools.link as pool_link, pools.slug as pool_slug,
pools.addresses as pool_addresses, pools.regexes as pool_regexes,
previous_block_hash as previousblockhash
FROM blocks
JOIN pools ON blocks.pool_id = pools.id
WHERE hash = '${hash}';
`;
const [rows]: any[] = await DB.query(query);
if (rows.length <= 0) {
return null;
}
rows[0].fee_span = JSON.parse(rows[0].fee_span);
return rows[0];
} catch (e) {
logger.err(`Cannot get indexed block ${hash}. Reason: ` + (e instanceof Error ? e.message : e));
throw e;
}
}
/**
* Return blocks difficulty
*/
@@ -397,18 +479,28 @@ class BlocksRepository {
const [blocks]: any[] = await DB.query(`SELECT height, hash, previous_block_hash,
UNIX_TIMESTAMP(blockTimestamp) as timestamp FROM blocks ORDER BY height`);
let currentHeight = 1;
while (currentHeight < blocks.length) {
if (blocks[currentHeight].previous_block_hash !== blocks[currentHeight - 1].hash) {
logger.warn(`Chain divergence detected at block ${blocks[currentHeight - 1].height}, re-indexing newer blocks and hashrates`);
await this.$deleteBlocksFrom(blocks[currentHeight - 1].height);
await HashratesRepository.$deleteHashratesFromTimestamp(blocks[currentHeight - 1].timestamp - 604800);
let partialMsg = false;
let idx = 1;
while (idx < blocks.length) {
if (blocks[idx].height - 1 !== blocks[idx - 1].height) {
if (partialMsg === false) {
logger.info('Some blocks are not indexed, skipping missing blocks during chain validation');
partialMsg = true;
}
++idx;
continue;
}
if (blocks[idx].previous_block_hash !== blocks[idx - 1].hash) {
logger.warn(`Chain divergence detected at block ${blocks[idx - 1].height}, re-indexing newer blocks and hashrates`);
await this.$deleteBlocksFrom(blocks[idx - 1].height);
await HashratesRepository.$deleteHashratesFromTimestamp(blocks[idx - 1].timestamp - 604800);
return false;
}
++currentHeight;
++idx;
}
logger.info(`${currentHeight} blocks hash validated in ${new Date().getTime() - start} ms`);
logger.info(`${idx} blocks hash validated in ${new Date().getTime() - start} ms`);
return true;
} catch (e) {
logger.err('Cannot validate chain of block hash. Reason: ' + (e instanceof Error ? e.message : e));
@@ -435,9 +527,9 @@ class BlocksRepository {
public async $getHistoricalBlockFees(div: number, interval: string | null): Promise<any> {
try {
let query = `SELECT
CAST(AVG(height) as INT) as avg_height,
CAST(AVG(height) as INT) as avgHeight,
CAST(AVG(UNIX_TIMESTAMP(blockTimestamp)) as INT) as timestamp,
CAST(AVG(fees) as INT) as avg_fees
CAST(AVG(fees) as INT) as avgFees
FROM blocks`;
if (interval !== null) {
@@ -457,12 +549,12 @@ class BlocksRepository {
/**
* Get the historical averaged block rewards
*/
public async $getHistoricalBlockRewards(div: number, interval: string | null): Promise<any> {
public async $getHistoricalBlockRewards(div: number, interval: string | null): Promise<any> {
try {
let query = `SELECT
CAST(AVG(height) as INT) as avg_height,
CAST(AVG(height) as INT) as avgHeight,
CAST(AVG(UNIX_TIMESTAMP(blockTimestamp)) as INT) as timestamp,
CAST(AVG(reward) as INT) as avg_rewards
CAST(AVG(reward) as INT) as avgRewards
FROM blocks`;
if (interval !== null) {
@@ -485,15 +577,15 @@ class BlocksRepository {
public async $getHistoricalBlockFeeRates(div: number, interval: string | null): Promise<any> {
try {
let query = `SELECT
CAST(AVG(height) as INT) as avg_height,
CAST(AVG(height) as INT) as avgHeight,
CAST(AVG(UNIX_TIMESTAMP(blockTimestamp)) as INT) as timestamp,
CAST(AVG(JSON_EXTRACT(fee_span, '$[0]')) as INT) as avg_fee_0,
CAST(AVG(JSON_EXTRACT(fee_span, '$[1]')) as INT) as avg_fee_10,
CAST(AVG(JSON_EXTRACT(fee_span, '$[2]')) as INT) as avg_fee_25,
CAST(AVG(JSON_EXTRACT(fee_span, '$[3]')) as INT) as avg_fee_50,
CAST(AVG(JSON_EXTRACT(fee_span, '$[4]')) as INT) as avg_fee_75,
CAST(AVG(JSON_EXTRACT(fee_span, '$[5]')) as INT) as avg_fee_90,
CAST(AVG(JSON_EXTRACT(fee_span, '$[6]')) as INT) as avg_fee_100
CAST(AVG(JSON_EXTRACT(fee_span, '$[0]')) as INT) as avgFee_0,
CAST(AVG(JSON_EXTRACT(fee_span, '$[1]')) as INT) as avgFee_10,
CAST(AVG(JSON_EXTRACT(fee_span, '$[2]')) as INT) as avgFee_25,
CAST(AVG(JSON_EXTRACT(fee_span, '$[3]')) as INT) as avgFee_50,
CAST(AVG(JSON_EXTRACT(fee_span, '$[4]')) as INT) as avgFee_75,
CAST(AVG(JSON_EXTRACT(fee_span, '$[5]')) as INT) as avgFee_90,
CAST(AVG(JSON_EXTRACT(fee_span, '$[6]')) as INT) as avgFee_100
FROM blocks`;
if (interval !== null) {
@@ -516,9 +608,9 @@ class BlocksRepository {
public async $getHistoricalBlockSizes(div: number, interval: string | null): Promise<any> {
try {
let query = `SELECT
CAST(AVG(height) as INT) as avg_height,
CAST(AVG(height) as INT) as avgHeight,
CAST(AVG(UNIX_TIMESTAMP(blockTimestamp)) as INT) as timestamp,
CAST(AVG(size) as INT) as avg_size
CAST(AVG(size) as INT) as avgSize
FROM blocks`;
if (interval !== null) {
@@ -541,9 +633,9 @@ class BlocksRepository {
public async $getHistoricalBlockWeights(div: number, interval: string | null): Promise<any> {
try {
let query = `SELECT
CAST(AVG(height) as INT) as avg_height,
CAST(AVG(height) as INT) as avgHeight,
CAST(AVG(UNIX_TIMESTAMP(blockTimestamp)) as INT) as timestamp,
CAST(AVG(weight) as INT) as avg_weight
CAST(AVG(weight) as INT) as avgWeight
FROM blocks`;
if (interval !== null) {

3
backend/src/repositories/HashratesRepository.ts
View File

@@ -1,3 +1,4 @@
import { escape } from 'mysql2';
import { Common } from '../api/common';
import DB from '../database';
import logger from '../logger';
@@ -105,7 +106,7 @@ class HashratesRepository {
public async $getPoolWeeklyHashrate(slug: string): Promise<any[]> {
const pool = await PoolsRepository.$getPool(slug);
if (!pool) {
throw new Error(`This mining pool does not exist`);
throw new Error('This mining pool does not exist ' + escape(slug));
}
// Find hashrate boundaries

1
backend/src/repositories/PoolsRepository.ts
View File

@@ -78,7 +78,6 @@ class PoolsRepository {
const [rows]: any[] = await DB.query(query, [slug]);
if (rows.length < 1) {
logger.debug(`This slug does not match any known pool`);
return null;
}

21
backend/src/repositories/RatesRepository.ts Normal file
View File

@@ -0,0 +1,21 @@
import DB from '../database';
import logger from '../logger';
import { IConversionRates } from '../mempool.interfaces';
class RatesRepository {
public async $saveRate(height: number, rates: IConversionRates) {
try {
await DB.query(`INSERT INTO rates(height, bisq_rates) VALUE (?, ?)`, [height, JSON.stringify(rates)]);
} catch (e: any) {
if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
logger.debug(`Rate already exists for block ${height}, ignoring`);
} else {
logger.err(`Cannot save exchange rate into db for block ${height} Reason: ` + (e instanceof Error ? e.message : e));
throw e;
}
}
}
}
export default new RatesRepository();

80
backend/src/routes.ts
View File

@@ -572,9 +572,9 @@ class Routes {
}
}
public async $getPools(interval: string, req: Request, res: Response) {
public async $getPools(req: Request, res: Response) {
try {
const stats = await miningStats.$getPoolsStats(interval);
const stats = await miningStats.$getPoolsStats(req.params.interval);
const blockCount = await BlocksRepository.$blockCount(null, null);
res.header('Pragma', 'public');
res.header('Cache-control', 'public');
@@ -588,7 +588,7 @@ class Routes {
public async $getPoolsHistoricalHashrate(req: Request, res: Response) {
try {
const hashrates = await HashratesRepository.$getPoolsWeeklyHashrate(req.params.interval ?? null);
const hashrates = await HashratesRepository.$getPoolsWeeklyHashrate(req.params.interval);
const blockCount = await BlocksRepository.$blockCount(null, null);
res.header('Pragma', 'public');
res.header('Cache-control', 'public');
@@ -619,9 +619,17 @@ class Routes {
}
public async $getHistoricalHashrate(req: Request, res: Response) {
let currentHashrate = 0, currentDifficulty = 0;
try {
const hashrates = await HashratesRepository.$getNetworkDailyHashrate(req.params.interval ?? null);
const difficulty = await BlocksRepository.$getBlocksDifficulty(req.params.interval ?? null);
currentHashrate = await bitcoinClient.getNetworkHashPs();
currentDifficulty = await bitcoinClient.getDifficulty();
} catch (e) {
logger.debug('Bitcoin Core is not available, using zeroed value for current hashrate and difficulty');
}
try {
const hashrates = await HashratesRepository.$getNetworkDailyHashrate(req.params.interval);
const difficulty = await BlocksRepository.$getBlocksDifficulty(req.params.interval);
const blockCount = await BlocksRepository.$blockCount(null, null);
res.header('Pragma', 'public');
res.header('Cache-control', 'public');
@@ -630,8 +638,8 @@ class Routes {
res.json({
hashrates: hashrates,
difficulty: difficulty,
currentHashrate: await bitcoinClient.getNetworkHashPs(),
currentDifficulty: await bitcoinClient.getDifficulty(),
currentHashrate: currentHashrate,
currentDifficulty: currentDifficulty,
});
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
@@ -640,12 +648,12 @@ class Routes {
public async $getHistoricalBlockFees(req: Request, res: Response) {
try {
const blockFees = await mining.$getHistoricalBlockFees(req.params.interval ?? null);
const blockFees = await mining.$getHistoricalBlockFees(req.params.interval);
const blockCount = await BlocksRepository.$blockCount(null, null);
res.header('Pragma', 'public');
res.header('Cache-control', 'public');
res.header('X-total-count', blockCount.toString());
res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString());
res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
res.json(blockFees);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
@@ -654,12 +662,12 @@ class Routes {
public async $getHistoricalBlockRewards(req: Request, res: Response) {
try {
const blockRewards = await mining.$getHistoricalBlockRewards(req.params.interval ?? null);
const blockRewards = await mining.$getHistoricalBlockRewards(req.params.interval);
const blockCount = await BlocksRepository.$blockCount(null, null);
res.header('Pragma', 'public');
res.header('Cache-control', 'public');
res.header('X-total-count', blockCount.toString());
res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString());
res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
res.json(blockRewards);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
@@ -668,15 +676,13 @@ class Routes {
public async $getHistoricalBlockFeeRates(req: Request, res: Response) {
try {
const blockFeeRates = await mining.$getHistoricalBlockFeeRates(req.params.interval ?? null);
const oldestIndexedBlockTimestamp = await BlocksRepository.$oldestBlockTimestamp();
const blockFeeRates = await mining.$getHistoricalBlockFeeRates(req.params.interval);
const blockCount = await BlocksRepository.$blockCount(null, null);
res.header('Pragma', 'public');
res.header('Cache-control', 'public');
res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString());
res.json({
oldestIndexedBlockTimestamp: oldestIndexedBlockTimestamp,
blockFeeRates: blockFeeRates,
});
res.header('X-total-count', blockCount.toString());
res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
res.json(blockFeeRates);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
}
@@ -684,13 +690,13 @@ class Routes {
public async $getHistoricalBlockSizeAndWeight(req: Request, res: Response) {
try {
const blockSizes = await mining.$getHistoricalBlockSizes(req.params.interval ?? null);
const blockWeights = await mining.$getHistoricalBlockWeights(req.params.interval ?? null);
const blockSizes = await mining.$getHistoricalBlockSizes(req.params.interval);
const blockWeights = await mining.$getHistoricalBlockWeights(req.params.interval);
const blockCount = await BlocksRepository.$blockCount(null, null);
res.header('Pragma', 'public');
res.header('Cache-control', 'public');
res.header('X-total-count', blockCount.toString());
res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString());
res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
res.json({
sizes: blockSizes,
weights: blockWeights
@@ -702,8 +708,9 @@ class Routes {
public async getBlock(req: Request, res: Response) {
try {
const result = await bitcoinApi.$getBlock(req.params.hash);
res.json(result);
const block = await blocks.$getBlock(req.params.hash);
res.setHeader('Expires', new Date(Date.now() + 1000 * 600).toUTCString());
res.json(block);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
}
@@ -719,19 +726,22 @@ class Routes {
}
}
public async getBlocksExtras(req: Request, res: Response) {
public async getBlocks(req: Request, res: Response) {
try {
const height = req.params.height === undefined ? undefined : parseInt(req.params.height, 10);
res.json(await blocks.$getBlocksExtras(height, 15));
if (['mainnet', 'testnet', 'signet', 'regtest'].includes(config.MEMPOOL.NETWORK)) { // Bitcoin
const height = req.params.height === undefined ? undefined : parseInt(req.params.height, 10);
res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
res.json(await blocks.$getBlocks(height, 15));
} else { // Liquid, Bisq
return await this.getLegacyBlocks(req, res);
}
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
}
}
public async getBlocks(req: Request, res: Response) {
try {
loadingIndicators.setProgress('blocks', 0);
public async getLegacyBlocks(req: Request, res: Response) {
try {
const returnBlocks: IEsploraApi.Block[] = [];
const fromHeight = parseInt(req.params.height, 10) || blocks.getCurrentBlockHeight();
@@ -755,16 +765,15 @@ class Routes {
returnBlocks.push(block);
nextHash = block.previousblockhash;
}
loadingIndicators.setProgress('blocks', i / 10 * 100);
}
res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
res.json(returnBlocks);
} catch (e) {
loadingIndicators.setProgress('blocks', 100);
res.status(500).send(e instanceof Error ? e.message : e);
}
}
public async getBlockTransactions(req: Request, res: Response) {
try {
loadingIndicators.setProgress('blocktxs-' + req.params.hash, 0);
@@ -776,9 +785,9 @@ class Routes {
const endIndex = Math.min(startingIndex + 10, txIds.length);
for (let i = startingIndex; i < endIndex; i++) {
try {
const transaction = await transactionUtils.$getTransactionExtended(txIds[i], true);
const transaction = await transactionUtils.$getTransactionExtended(txIds[i], true, true);
transactions.push(transaction);
loadingIndicators.setProgress('blocktxs-' + req.params.hash, (i + 1) / endIndex * 100);
loadingIndicators.setProgress('blocktxs-' + req.params.hash, (i - startingIndex + 1) / (endIndex - startingIndex) * 100);
} catch (e) {
logger.debug('getBlockTransactions error: ' + (e instanceof Error ? e.message : e));
}
@@ -1001,6 +1010,7 @@ class Routes {
public async $getRewardStats(req: Request, res: Response) {
try {
const response = await mining.$getRewardStats(parseInt(req.params.blockCount, 10));
res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
res.json(response);
} catch (e) {
res.status(500).end();

96
backend/src/tasks/pools-updater.ts
View File

@@ -1,8 +1,10 @@
const https = require('https');
import axios from 'axios';
import poolsParser from '../api/pools-parser';
import config from '../config';
import DB from '../database';
import logger from '../logger';
import { SocksProxyAgent } from 'socks-proxy-agent';
import * as https from 'https';
/**
* Maintain the most recent version of pools.json
@@ -14,7 +16,7 @@ class PoolsUpdater {
}
public async updatePoolsJson() {
if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) {
if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false || config.DATABASE.ENABLED === false) {
return;
}
@@ -28,6 +30,13 @@ class PoolsUpdater {
this.lastRun = now;
logger.info('Updating latest mining pools from Github');
if (config.SOCKS5PROXY.ENABLED) {
logger.info('List of public pools will be queried over the Tor network');
} else {
logger.info('List of public pools will be queried over clearnet');
}
try {
const dbSha = await this.getShaFromDb();
const githubSha = await this.fetchPoolsSha(); // Fetch pools.json sha from github
@@ -41,7 +50,10 @@ class PoolsUpdater {
}
logger.warn('Pools.json is outdated, fetch latest from github');
const poolsJson = await this.fetchPools();
const poolsJson = await this.query('https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json');
if (poolsJson === undefined) {
return;
}
await poolsParser.migratePoolsJson(poolsJson);
await this.updateDBSha(githubSha);
logger.notice('PoolsUpdater completed');
@@ -52,14 +64,6 @@ class PoolsUpdater {
}
}
/**
* Fetch pools.json from github repo
*/
private async fetchPools(): Promise<object> {
const response = await this.query('/repos/mempool/mining-pools/contents/pools.json');
return JSON.parse(Buffer.from(response['content'], 'base64').toString('utf8'));
}
/**
* Fetch our latest pools.json sha from the db
*/
@@ -90,11 +94,13 @@ class PoolsUpdater {
* Fetch our latest pools.json sha from github
*/
private async fetchPoolsSha(): Promise<string | undefined> {
const response = await this.query('/repos/mempool/mining-pools/git/trees/master');
const response = await this.query('https://api.github.com/repos/mempool/mining-pools/git/trees/master');
for (const file of response['tree']) {
if (file['path'] === 'pools.json') {
return file['sha'];
if (response !== undefined) {
for (const file of response['tree']) {
if (file['path'] === 'pools.json') {
return file['sha'];
}
}
}
@@ -105,35 +111,45 @@ class PoolsUpdater {
/**
* Http request wrapper
*/
private query(path): Promise<string> {
return new Promise((resolve, reject) => {
const options = {
host: 'api.github.com',
path: path,
method: 'GET',
headers: { 'user-agent': 'node.js' }
private async query(path): Promise<object | undefined> {
type axiosOptions = {
httpsAgent?: https.Agent;
}
const setDelay = (secs: number = 1): Promise<void> => new Promise(resolve => setTimeout(() => resolve(), secs * 1000));
const axiosOptions: axiosOptions = {};
let retry = 0;
if (config.SOCKS5PROXY.ENABLED) {
const socksOptions: any = {
agentOptions: {
keepAlive: true,
},
hostname: config.SOCKS5PROXY.HOST,
port: config.SOCKS5PROXY.PORT
};
logger.debug('Querying: api.github.com' + path);
if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) {
socksOptions.username = config.SOCKS5PROXY.USERNAME;
socksOptions.password = config.SOCKS5PROXY.PASSWORD;
}
const request = https.get(options, (response) => {
const chunks_of_data: any[] = [];
response.on('data', (fragments) => {
chunks_of_data.push(fragments);
});
response.on('end', () => {
resolve(JSON.parse(Buffer.concat(chunks_of_data).toString()));
});
response.on('error', (error) => {
reject(error);
});
});
axiosOptions.httpsAgent = new SocksProxyAgent(socksOptions);
}
request.on('error', (error) => {
logger.err('Github API query failed. Reason: ' + error);
reject(error);
});
});
while(retry < 5) {
try {
const data = await axios.get(path, axiosOptions);
if (data.statusText !== 'OK' || !data.data) {
throw new Error(`Could not fetch data from Github, Error: ${data.status}`);
}
return data.data;
} catch (e) {
logger.err('Could not connect to Github. Reason: ' + (e instanceof Error ? e.message : e));
retry++;
}
await setDelay();
}
return undefined;
}
}

10
backend/src/utils/blocks-utils.ts
View File

@@ -1,4 +1,4 @@
import { BlockExtended } from "../mempool.interfaces";
import { BlockExtended } from '../mempool.interfaces';
export function prepareBlock(block: any): BlockExtended {
return <BlockExtended>{
@@ -15,11 +15,13 @@ export function prepareBlock(block: any): BlockExtended {
weight: block.weight,
previousblockhash: block.previousblockhash,
extras: {
coinbaseRaw: block.coinbase_raw ?? block.extras.coinbaseRaw,
coinbaseRaw: block.coinbase_raw ?? block.extras?.coinbaseRaw,
medianFee: block.medianFee ?? block.median_fee ?? block.extras?.medianFee,
feeRange: block.feeRange ?? block.fee_range ?? block?.extras?.feeSpan,
feeRange: block.feeRange ?? block.fee_span,
reward: block.reward ?? block?.extras?.reward,
totalFees: block.totalFees ?? block?.fees ?? block?.extras.totalFees,
totalFees: block.totalFees ?? block?.fees ?? block?.extras?.totalFees,
avgFee: block?.extras?.avgFee ?? block.avg_fee,
avgFeeRate: block?.avgFeeRate ?? block.avg_fee_rate,
pool: block?.extras?.pool ?? (block?.pool_id ? {
id: block.pool_id,
name: block.pool_name,